Android10 AudioRecord简单解析
AudioRecord简介:
音频子系统,从音频输入设备录制音频PCM数据供应用层使用
业务场景:蓝牙语音遥控器，带麦克风的摄像头，耳机等。应用拿到音频PCM数据可以直接播放(比如常见的录音apk，录音达人)
或者对语音进行解析(语音转文字,语音助手应用)
本文是对国科平台android10进行梳理，其它版本SDK可能有所差异，但总体框架是类似的
涉及源码路径:
AudioRecord.java(frameworks/base/media/java/android/media/AudioRecord.java)
android_media_AudioRecord.cpp(frameworks/base/core/jni/android_media_AudioRecord.cpp)
AudioRecord.cpp(frameworks/av/media/libaudioclient/AudioRecord.cpp)
AudioSystem.cpp(frameworks/av/media/libaudioclient/AudioSystem.cpp)
AudioFlinger.cpp(frameworks/av/services/audioflinger/AudioFlinger.cpp)
AudioPolicyService.cpp(frameworks/av/services/audiopolicy/service/AudioPolicyService.cpp)
AudioPolicyInterfaceImpl.cpp(frameworks/av/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp)
AudioPolicyManager.cpp(frameworks/av/services/audiopolicy/managerdefault/AudioPolicyManager.cpp)
Engine.cpp(frameworks/av/services/audiopolicy/enginedefault/src/Engine.cpp)
AudioInputDescriptor.cpp(frameworks/av/services/audiopolicy/common/managerdefinitions/src/AudioInputDescriptor.cpp)
AudioPolicyClientImpl.cpp(frameworks/av/services/audiopolicy/service/AudioPolicyClientImpl.cpp)
main_audioserver.cpp(frameworks/av/media/audioserver/main_audioserver.cpp)
DeviceHalLocal.cpp(frameworks/av/media/libaudiohal/impl/DeviceHalLocal.cpp)
Threads.cpp(frameworks/av/services/audioflinger/Threads.cpp)
概述:
AudioRecord：audio系统对外的API类,负责音频数据采集(录音)
AudioFlinger: audio系统的工作引擎,管理输入输出流,和底层音频相关的硬件交互
AudioPolicyService: 音频策略服务,管理音频设备的选择和切换，管理音量等
1.AudioRecord用例介绍:
int recordBufferSize = AudioRecord.getMinBufferSize(16000,//采样率
AudioFormat.CHANNEL_IN_STEREO,//声道数,双声道
AudioFormat.ENCODING_PCM_16BIT//采样精度，一个采样点16bit，两个字节
);
//②创建AudioRecord
AudioRecord audioRecord = new AudioRecord(MediaRecorder.AudioSource.MIC，//输入源，麦克风类型
16000，
AudioFormat.CHANNEL_IN_STEREO，
AudioFormat.ENCODING_PCM_16BIT，
recordBufferSize)；
//③启动录音
audioRecord.startRecording();
//④读取录音数据
audioRecord.read(data, 0, recordBufferSize);
.......
//⑤停止录音，释放底层资源
audioRecord.stop()；
audioRecord.release();Frame(帧):描述数据量大小，比如一帧等于多少字节, 在音频系统中 1单位Frame等于1个采样点字节数 X 声道数
(比如上面的采样精度 16BIT,双声道，那么一Frame就是 2X2=4字节)
最小缓冲区大小=最低帧数∗声道数∗采样精度
(最小缓冲区大小也和底层硬件有关，比如硬件是否支持当前采样率，采样精度，硬件延时等情况，最后综合计算出一个大小)
2.AudioRecord分析(java层)
AudioRecord.java
public AudioRecord(int audioSource, int sampleRateInHz, int channelConfig, int audioFormat,
int bufferSizeInBytes){
//这个会调用下面这个同名重载的构造函数
}
public AudioRecord(AudioAttributes attributes, AudioFormat format, int bufferSizeInBytes,
int sessionId) throws IllegalArgumentException {
//检查参数是否合法
audioParamCheck(attributes.getCapturePreset(), rate, encoding);
audioBuffSizeCheck(bufferSizeInBytes);
......
//到native层
int initResult = native_setup( new WeakReference<AudioRecord>(this),
mAudioAttributes, sampleRate, mChannelMask, mChannelIndexMask,
mAudioFormat, mNativeBufferSizeInBytes,
session, getCurrentOpPackageName(), 0 /*nativeRecordInJavaObj*/);
}android_media_AudioRecord.cpp
static jint
android_media_AudioRecord_setup(JNIEnv *env, jobject thiz, jobject weak_this,
jobject jaa, jintArray jSampleRate, jint channelMask, jint channelIndexMask,
jint audioFormat, jint buffSizeInBytes, jintArray jSession, jstring opPackageName,
jlong nativeRecordInJavaObj)
{
//前面主要是做了一些java层值到jni层值的转换
......
//创建native层的AudioRecord
lpRecorder = new AudioRecord(String16(opPackageNameStr.c_str()));
......
//调用 AudioRecord::set()
const status_t status = lpRecorder->set(paa->source,
sampleRateInHertz,
format, // word length, PCM
localChanMask,
frameCount,
recorderCallback,// callback_t
lpCallbackData,// void* user
0, // notificationFrames,
true, // threadCanCallJava
sessionId,
AudioRecord::TRANSFER_DEFAULT,
flags,
-1, -1, // default uid, pid
paa.get());
//native层AudioRecord保存到java层的mNativeRecorderInJavaObj变量中
setAudioRecord(env, thiz, lpRecorder);
}java层AudioRecord总结:
在构造函数中调用native层接口，创建native层的AudioRecord，后续的startRecording,read,stop,release等操作都是通过这个native层的AudioRecord去处理
3.AudioRecord分析(native层)
AudioRecord.cpp
//构造函数中对成员变量进行初始化
AudioRecord::AudioRecord(const String16 &opPackageName)
: mActive(false), mStatus(NO_INIT), mOpPackageName(opPackageName),
mSessionId(AUDIO_SESSION_ALLOCATE),
mPreviousPriority(ANDROID_PRIORITY_NORMAL), mPreviousSchedulingGroup(SP_DEFAULT),
mSelectedDeviceId(AUDIO_PORT_HANDLE_NONE), mRoutedDeviceId(AUDIO_PORT_HANDLE_NONE),
mSelectedMicDirection(MIC_DIRECTION_UNSPECIFIED),
mSelectedMicFieldDimension(MIC_FIELD_DIMENSION_DEFAULT)
{
}
status_t AudioRecord::set(audio_source_t inputSource, uint32_t sampleRate,
audio_format_t format, audio_channel_mask_t channelMask, size_t frameCount,
callback_t cbf, void* user, uint32_t notificationFrames,
bool threadCanCallJava, audio_session_t sessionId,transfer_type transferType,
audio_input_flags_t flags, uid_t uid, pid_t pid,
const audio_attributes_t* pAttributes, audio_port_handle_t selectedDeviceId,
audio_microphone_direction_t selectedMicDirection, float microphoneFieldDimension)
{
//如果有回调函数,会启动AudioRecordThread 处理回调函数
if (cbf != NULL) {
mAudioRecordThread = new AudioRecordThread(*this);
mAudioRecordThread->run("AudioRecord", ANDROID_PRIORITY_AUDIO);
}
// create the IAudioRecord
status = createRecord_l(0 /*epoch*/, mOpPackageName);
......
}
status_t AudioRecord::createRecord_l(const Modulo<uint32_t> &epoch, const String16& opPackageName)
{
......
//IAudioRecord record 通过Binder调用到AudioFlinger去初始化音频服务里录音相关的流程
record = audioFlinger->createRecord(input, output, &status);
......
//audio_track_cblk_t* cblk 共享内存，
//音频数据从音频服务进程(AudioFlinger所在的进程)到客户端进程,通过共享内存来实现
//AudioRecordClientProxy类定义了一些操作共享内存的方法
mProxy = new AudioRecordClientProxy(cblk, buffers, mFrameCount, mFrameSize);
}native层AudioRecord总结:
native层流程里主要拿到了两个客户端资源，一个是record,能和AudioFlinger里的一个RecordTrack类型的track交互，控制录音启动，停止等
从底层硬件采集音频数据的工作是在音频服务进程中进行的，客户端拿到的这个record能通过远程调用的方式对录音流程进行控制
IAudioRecord是联系AudioRecord和AudioFlinger的纽带,IAudioRecord在AudioFlinger那边具体是什么,在后续的流程中会有解释
AudioRecord和AudioFlinger交互流程图
4.AudioFlinger分析
AudioFlinger驻留在audioserver进程中
main_audioserver.cpp
int main(int argc __unused, char **argv)
{
......
sp<ProcessState> proc(ProcessState::self());
sp<IServiceManager> sm = defaultServiceManager();
//注册AudioFlinger和AudioPolicyService服务
AudioFlinger::instantiate();
AudioPolicyService::instantiate();
......
ProcessState::self()->startThreadPool();
IPCThreadState::self()->joinThreadPool();
}
}AudioFlinger.cpp
sp<media::IAudioRecord> AudioFlinger::createRecord(const CreateRecordInput& input,
CreateRecordOutput& output,
status_t *status)
{
sp<RecordThread::RecordTrack> recordTrack;
sp<RecordHandle> recordHandle;
......
//初始化open输入流 启动Record线程
lStatus = AudioSystem::getInputForAttr(&input.attr, &output.inputId,
input.riid,
sessionId,
clientPid,
clientUid,
input.opPackageName,
&input.config,
output.flags, &output.selectedDeviceId, &portId);
......
//output.inputId是一个索引号, 根据索引号可以获取一个录音用的工作线程 RecordThread
RecordThread *thread = checkRecordThread_l(output.inputId);
......
//创建线程对应的recordTrack
recordTrack = thread->createRecordTrack_l(client, input.attr, &output.sampleRate,
input.config.format, input.config.channel_mask, &output.frameCount,
sessionId, &output.notificationFrameCount, callingPid, clientUid,
&output.flags, input.clientInfo.clientTid, &lStatus, portId,
input.opPackageName);
......
// RecordHandle继承BnAudioRecord实现了服务端,可对recordTrack进行一些流程上的控制(start stop啥的)
recordHandle = new RecordHandle(recordTrack);
......
//当recordHandle通过Binder服务被从 AudioFlinger送到 AudioRecord时， AudioRecord那边拿到的其实是客户端BpAudioRecord,这个和匿名Binder有关,详细可自行了解
return recordHandle;
}AudioSystem.cpp
AudioSystem::getInputForAttr()流程跟踪
status_t AudioSystem::getInputForAttr(const audio_attributes_t *attr,
audio_io_handle_t *input, audio_unique_id_t riid,
audio_session_t session, pid_t pid, uid_t uid,
const String16& opPackageName, const audio_config_base_t *config,
audio_input_flags_t flags, audio_port_handle_t *selectedDeviceId,
audio_port_handle_t *portId)
{
//AudioSystem里的接口基本都是这样,封装了对AudioPolicyService(或者AudioFlinger)的远程调用，方便其它模块访问使用服务的功能
const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service();
if (aps == 0) return NO_INIT;
return aps->getInputForAttr(attr, input, riid, session, pid, uid, opPackageName,config, flags, selectedDeviceId, portId);
}AudioPolicyInterfaceImpl.cpp 中实现了许多AudioPolicyService的接口
status_t AudioPolicyService::getInputForAttr(const audio_attributes_t *attr,
audio_io_handle_t *input, audio_unique_id_t riid,
audio_session_t session,pid_t pid, uid_t uid,
const String16& opPackageName, const audio_config_base_t *config,
audio_input_flags_t flags, audio_port_handle_t *selectedDeviceId,
audio_port_handle_t *portId)
{
......
//检查客户端进程是否有录音权限 android.permission.RECORD_AUDIO
if (!recordingAllowed(opPackageName, pid, uid)){
......
}
//android.permission.CAPTURE_AUDIO_OUTPUT 比如通话录音情景
bool canCaptureOutput = captureAudioOutputAllowed(pid, uid);
......
//到AudioPolicyManager去, mAudioPolicyManager在AudioPolicyService构造函数中被初始化
status = mAudioPolicyManager->getInputForAttr(attr, input, riid, session, uid, config, flags, selectedDeviceId,
&inputType, portId);
......
}AudioPolicyManager.cpp
status_t AudioPolicyManager::getInputForAttr(const audio_attributes_t *attr,
audio_io_handle_t *input, audio_unique_id_t riid,
audio_session_t session, uid_t uid,
const audio_config_base_t *config, audio_input_flags_t flags,
audio_port_handle_t *selectedDeviceId, input_type_t *inputType,
audio_port_handle_t *portId)
{
...... //忽略前面的一系列判断，一般情况会走到这里
//通过attributes 来拿到符合的输入设备
device = mEngine->getInputDeviceForAttributes(attributes, &policyMix);
//通过输入设备和其它的音频参数，拿到一个符合的输入流
*input = getInputForDevice(device, session, attributes, config, flags, policyMix);
}Engine.cpp
sp<DeviceDescriptor> Engine::getInputDeviceForAttributes(const audio_attributes_t &attr,
sp<AudioPolicyMix> *mix) const
{
......
//用source类型拿到一个输入设备的类型, 我们这里是 AUDIO_SOURCE_MIC
audio_devices_t deviceType = getDeviceForInputSource(attr.source);
......
//availableInputDevices是当前活跃(可用)的所有输入设备列表
return availableInputDevices.getDevice(deviceType,
String8(address.c_str()),
AUDIO_FORMAT_DEFAULT);
}
audio_devices_t Engine::getDeviceForInputSource(audio_source_t inputSource) const
{
......
switch (inputSource) {
case AUDIO_SOURCE_DEFAULT:
case AUDIO_SOURCE_MIC:
if (availableDeviceTypes & AUDIO_DEVICE_IN_BLUETOOTH_A2DP) {
device = AUDIO_DEVICE_IN_BLUETOOTH_A2DP;
} else if ((getForceUse(AUDIO_POLICY_FORCE_FOR_RECORD) == AUDIO_POLICY_FORCE_BT_SCO) &&
(availableDeviceTypes & AUDIO_DEVICE_IN_BLUETOOTH_SCO_HEADSET)) {
device = AUDIO_DEVICE_IN_BLUETOOTH_SCO_HEADSET;
} else if (availableDeviceTypes & AUDIO_DEVICE_IN_WIRED_HEADSET) {
device = AUDIO_DEVICE_IN_WIRED_HEADSET;
} else if (availableDeviceTypes & AUDIO_DEVICE_IN_USB_HEADSET) {
device = AUDIO_DEVICE_IN_USB_HEADSET;
} else if (availableDeviceTypes & AUDIO_DEVICE_IN_USB_DEVICE) {
device = AUDIO_DEVICE_IN_USB_DEVICE;
} else if (availableDeviceTypes & AUDIO_DEVICE_IN_BUILTIN_MIC) {
//蓝牙语音遥控匹配到这个
device = AUDIO_DEVICE_IN_BUILTIN_MIC;
}
break;
.......
return device;
}拿到输入设备后再回到AudioPolicyManager::getInputForAttr()中,下一步是通过输入设备打开一个输入流
audio_io_handle_t AudioPolicyManager::getInputForDevice(const sp<DeviceDescriptor> &device,
audio_session_t session,
const audio_attributes_t &attributes,
const audio_config_base_t *config,
audio_input_flags_t flags,
const sp<AudioPolicyMix> &policyMix)
{
......
for (;;) {
//匹配audio_policy_configuration.xml中的解析到的 profile字段
//xml中描述定义了许多可用输入或者输出设备，AudioPolicyManager构造初始化时会解析该xml,并加载对应hal层so
//解析结果可通过 dumpsys media.audio_policy命令查看
profile = getInputProfile(device, profileSamplingRate,
profileFormat, profileChannelMask, profileFlags);
......
}
//根据匹配到的参数构造一个输入流描述, mpClientInterface类型是AudioPolicyClient,用于直接和AudioFlinger交互的
sp<AudioInputDescriptor> inputDesc = new AudioInputDescriptor(profile, mpClientInterface);
......
//去打开输入流, 最后会走到 hal层so对应的 adev_open_output_stream()
status_t status = inputDesc->open(&lConfig, device, halInputSource, profileFlags, &input);
...
return input;
}AudioInputDescriptor.cpp
status_t AudioInputDescriptor::open(const audio_config_t *config,
const sp<DeviceDescriptor> &device, audio_source_t source,
audio_input_flags_t flags, audio_io_handle_t *input)
{
......
//mpClientInterface类型是AudioPolicyClient, 也在AudioPolicyService构造函数中被初始化
//用于直接和AudioFlinger交互的
status_t status = mClientInterface->openInput(mProfile->getModuleHandle(),
input, &lConfig, &deviceType,
mDevice->address(), source, flags);
......
}AudioPolicyClientImpl.cpp
status_t AudioPolicyService::AudioPolicyClient::openInput(audio_module_handle_t module,
audio_io_handle_t *input, audio_config_t *config,
audio_devices_t *device, const String8& address,
audio_source_t source, audio_input_flags_t flags)
{
sp<IAudioFlinger> af = AudioSystem::get_audio_flinger();
......
//调用到AudioFlinger::openInput()
return af->openInput(module, input, config, device, address, source, flags);
}流程到这里,可以先稍微暂停总结一下，我们一开始在AudioFlinger::createRecord()中，然后调用AudioSystem::getInputForAttr()去拿一个输入流,但是流程经过一层又一层调用后，最后又回到了AudioFlinger。这也体现了android audio的设计框架，AudioPolicyService管理音频策略，它帮AudioFlinger去选了一个合适的输入设备，但最终打开设备，和底层audio模块交互，还得AudioFlinger自己来做,AudioFlinger是执行者。
AudioFlinger.cpp
status_t AudioFlinger::openInput(audio_module_handle_t module,
audio_io_handle_t *input, audio_config_t *config,
audio_devices_t *devices, const String8& address,
audio_source_t source, audio_input_flags_t flags)
{
......
//调用openInput_l() 打开输入流，启动一个thread
sp<ThreadBase> thread = openInput_l(
module, input, config, *devices, address, source, flags, AUDIO_DEVICE_NONE, String8{});
}
sp<AudioFlinger::ThreadBase> AudioFlinger::openInput_l(audio_module_handle_t module,
audio_io_handle_t *input, audio_config_t *config,
audio_devices_t devices, const String8& address,
audio_source_t source, audio_input_flags_t flags,
audio_devices_t outputDevice,
const String8& outputDeviceAddress)
{
......
//inHwHal类型是 DeviceHalLocal
status_t status = inHwHal->openInputStream(
*input, devices, &halconfig, flags, address.string(), source,
outputDevice, outputDeviceAddress, &inStream);
......
//RecordThread启动
sp<RecordThread> thread = new RecordThread(this,
inputStream,
*input,
primaryOutputDevice_l(),
devices,
mSystemReady
);
mRecordThreads.add(*input, thread);
return thread;
}DeviceHalLocal.cpp
status_t DeviceHalLocal::openInputStream(audio_io_handle_t handle,
audio_devices_t devices, struct audio_config *config,
audio_input_flags_t flags, const char *address,
audio_source_t source,audio_devices_t /*outputDevice*/,
const char */*outputDeviceAddress*/,
sp<StreamInHalInterface> *inStream) {
audio_stream_in_t *halStream;
//到对应的audio hal层 so中, open_input_stream()
int openResult = mDev->open_input_stream(
mDev, handle, devices, config, &halStream, flags, address, source);
}总结: AudioFlinger 调用AudioSystem::getInputForAttr()后会走到AudioPolicyService(音频策略服务)
策略服务综合各种条件(录音的参数, 可用输入设备, 结合实际场景选择一个合适的输入设备)。
AudioFlinger拿到输入设备后通过openInput() 去打开实际的输入流。auido hal层的so库，不同平台，不同项目会有差异
hardware/libhardware/modules/audio/audio_hw.c 是源码中默认的模板,可以看下代码结构
打开输入流，启动RecordThread后，还剩最后一个关键步骤，RecordTrack的创建:
Threads.cpp
sp<AudioFlinger::RecordThread::RecordTrack> AudioFlinger::RecordThread::createRecordTrack_l(
const sp<AudioFlinger::Client>& client, const audio_attributes_t& attr,
uint32_t *pSampleRate, audio_format_t format, audio_channel_mask_t channelMask,
size_t *pFrameCount, audio_session_t sessionId, size_t *pNotificationFrameCount,
pid_t creatorPid, uid_t uid, audio_input_flags_t *flags, pid_t tid, status_t *status,
audio_port_handle_t portId, const String16& opPackageName)
{
......
//RecordThread里的 RecordTrack
track = new RecordTrack(this, client, attr, sampleRate,
format, channelMask, frameCount,
nullptr /* buffer */, (size_t)0 /* bufferSize */, sessionId, creatorPid, uid,
*flags, TrackBase::TYPE_DEFAULT, opPackageName, portId);
......
return track;
}创建的RecordTrack会被打包进RecordHandle,RecordHandle对应客户端通过Binder发送给AudioRecord
这样AudioRecord就能和RecordTrack交互了，音频数据会通过RecordTrack送到AudioRecord