iOS音视频(音视频的编解码)

作者: 稀土掘金  更新时间:2020-03-28 22:26:06  原文链接


上一篇文章简单介绍了AVFoundation的使用,这篇文章就来看下怎么利用VideoToolbox和AudioToolbox去实现音视频的编解码。研究的视频流格式是H264,音频流格式是AAC,若不了解这两种格式,点击这里快速了解。

VideoToolbox

VideoToolbox 是一个低级的框架,可直接访问硬件的编解码器。能够为视频提供压缩和解压缩的服务,同时也提供存储在 CoreVideo 像素缓冲区的图像进行格式的转换。

首先来了解一下CMSampleBuffer数据结构,在视频采集过程中,数据结构有以下两种

可以看出,不同的地方是CMBlockBuffer和CVPixelBuffer。

CVPixelBuffer是未经过编码的原始数据,而CMBlockBuffer是经过编码的H264裸流数据,图示如下:

下面来用代码去实现这个流程

编码部分

初始化

#pragma mark - 编码
- (void)encodeInit{
    OSStatus status = VTCompressionSessionCreate(kCFAllocatorDefault, (uint32_t)_width, (uint32_t)_height, kCMVideoCodecType_H264, NULL, NULL, NULL, encodeCallBack, (__bridge void * _Nullable)(self), &_encoderSession);
    if (status != noErr) {
        return;
    }
    // 设置实时编码
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
    // 设置过滤B帧
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
    // 设置码率均值
    CFNumberRef bitRate = (__bridge CFNumberRef)(@(_height*1000));
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_AverageBitRate, bitRate);
    // 设置最值码率
    CFArrayRef limits = (__bridge CFArrayRef)(@[@(_height*1000/4),@(_height*1000*4)]);
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_DataRateLimits, limits);
    // 设置FPS
    CFNumberRef fps = (__bridge CFNumberRef)(@(25));
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_ExpectedFrameRate, fps);
    // 设置I帧间隔
    CFNumberRef maxKeyFrameInterval = (__bridge CFNumberRef)(@(25*2));
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, maxKeyFrameInterval);
    // 准备编码
    status = VTCompressionSessionPrepareToEncodeFrames(_encoderSession);
}

VTCompressionSessionCreate 参数解析

/*
 * allocator:内存分配器,填NULL为默认分配器
 * width、height:视频帧像素的宽高,如果编码器不支持这个宽高的话可能会改变
 * codecType:编码类型,枚举
 * encoderSpecification:指定特定的编码器,填NULL的话由VideoToolBox自动选择
 * sourceImageBufferAttributes:源像素缓冲区的属性,用于为源帧创建像素缓冲池。如果这个参数有值的话,VideoToolBox会创建一个缓冲池,不需要缓冲池可以设置为NULL。使用VideoToolbox没有分配的像素缓冲区可能会增加复制图像数据的机会。
 * compressedDataAllocator:压缩后数据的内存分配器,填NULL使用默认分配器
 * outputCallback:视频压缩后输出数据的回调函数。这个函数在调用VTCompressionSessionEncodeFrame的线程上被异步调用。NULL,为编码帧调用VTCompressionSessionEncodeFrameWithOutputHandler时调用。
 * param outputCallbackRefCon:回调函数中的自定义指针,我们通常传self,在回调函数中就可以拿到当前类的方法和属性了
 * compressionSessionOut:编码器句柄,传入编码器的指针
 */
VT_EXPORT OSStatus 
VTCompressionSessionCreate(
	CM_NULLABLE CFAllocatorRef							allocator,
	int32_t												width,
	int32_t												height,
	CMVideoCodecType									codecType,
	CM_NULLABLE CFDictionaryRef							encoderSpecification,
	CM_NULLABLE CFDictionaryRef							sourceImageBufferAttributes,
	CM_NULLABLE CFAllocatorRef							compressedDataAllocator,
	CM_NULLABLE VTCompressionOutputCallback				outputCallback,
	void * CM_NULLABLE									outputCallbackRefCon,
	CM_RETURNS_RETAINED_PARAMETER CM_NULLABLE VTCompressionSessionRef * CM_NONNULL compressionSessionOut) API_AVAILABLE(macosx(10.8), ios(8.0), tvos(10.2));

数据输入

- (void)encodeSampleBuffer:(CMSampleBufferRef)sampleBuffer{
    
    if (!self.encoderSession) {
        [self encodeInit];
    }
    
    CFRetain(sampleBuffer);
    dispatch_async(self.encoderQueue, ^{
        // 原始数据获取
        CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
        // 该帧的时间戳
        CMTime TimeStamp = CMTimeMake(self->frameID, 1000);
        self->frameID++;
        // 持续时间
        CMTime duration = kCMTimeInvalid;
        VTEncodeInfoFlags infoFlagsOut;
        // 编码
        OSStatus status = VTCompressionSessionEncodeFrame(self.encoderSession, imageBuffer, TimeStamp, duration, NULL, NULL, &infoFlagsOut);
        if (status != noErr) {
            NSLog(@"error");
        }
        CFRelease(sampleBuffer);
    });
}

编码回调方法

void encodeCallBack (
void * CM_NULLABLE outputCallbackRefCon,
void * CM_NULLABLE sourceFrameRefCon,
OSStatus status,
VTEncodeInfoFlags infoFlags,
CM_NULLABLE CMSampleBufferRef sampleBuffer ){
    if (status != noErr) {
        NSLog(@"encodeVideoCallBack: encode error, status = %d",(int)status);
        return;
    }
    
    if (!CMSampleBufferDataIsReady(sampleBuffer)) {
        NSLog(@"encodeVideoCallBack: data is not ready");
        return;
    }
    
    Demo2ViewController *VC = (__bridge Demo2ViewController *)(outputCallbackRefCon);
    
    // 判断是否为关键帧
    BOOL isKeyFrame = NO;
    CFArrayRef attachArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
    isKeyFrame = !CFDictionaryContainsKey(CFArrayGetValueAtIndex(attachArray, 0), kCMSampleAttachmentKey_NotSync);//(注意取反符号)
    
    if (isKeyFrame && !VC->hasSpsPps) {
        size_t spsSize, spsCount;
        size_t ppsSize, ppsCount;
        const uint8_t *spsData, *ppsData;
        
        // 获取图像源格式
        CMFormatDescriptionRef formatDesc = CMSampleBufferGetFormatDescription(sampleBuffer);
        OSStatus status1 = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(formatDesc, 0, &spsData, &spsSize, &spsCount, 0);
        OSStatus status2 = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(formatDesc, 1, &ppsData, &ppsSize, &ppsCount, 0);
        
        if (status1 == noErr && status2 == noErr) {
            VC->hasSpsPps = true;
            
            //sps data
            VC->sps = [NSMutableData dataWithCapacity:4 + spsSize];
            [VC->sps appendBytes:StartCode length:4];
            [VC->sps appendBytes:spsData length:spsSize];
            //pps data
            VC->pps = [NSMutableData dataWithCapacity:4 + ppsSize];
            [VC->pps appendBytes:StartCode length:4];
            [VC->pps appendBytes:ppsData length:ppsSize];
            
            [VC decodeH264Data:VC->sps];
            [VC decodeH264Data:VC->pps];
        }
    }
    
    // 获取NALU数据
    size_t lengthAtOffset, totalLength;
    char *dataPoint;
    
    // 将数据复制到dataPoint
    CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
    OSStatus error = CMBlockBufferGetDataPointer(blockBuffer, 0, &lengthAtOffset, &totalLength, &dataPoint);
    if (error != kCMBlockBufferNoErr) {
        NSLog(@"VideoEncodeCallback: get datapoint failed, status = %d", (int)error);
        return;
    }
    
    // 循环获取nalu数据
    size_t offet = 0;
    // 返回的nalu数据前四个字节不是0001的startcode(不是系统端的0001),而是大端模式的帧长度length
    const int lengthInfoSize = 4;
    while (offet < totalLength - lengthInfoSize) {
        uint32_t naluLength = 0;
        // 获取nalu 数据长度
        memcpy(&naluLength, dataPoint + offet, lengthInfoSize);
        // 大端转系统端
        naluLength = CFSwapInt32BigToHost(naluLength);
        // 获取到编码好的视频数据
        NSMutableData *data = [NSMutableData dataWithCapacity:4 + naluLength];
        [data appendBytes:StartCode length:4];
        [data appendBytes:dataPoint + offet + lengthInfoSize length:naluLength];
        
        dispatch_async(VC.encoderCallbackQueue, ^{
            [VC decodeH264Data:data];
        });
        
        // 移动下标,继续读取下一个数据
        offet += lengthInfoSize + naluLength;
    }
}

VTCompressionOutputCallback 参数解析

/*
 * outputCallbackRefCon:回调函数的引用值。
 * sourceFrameRefCon:帧的参考值,从sourceFrameRefCon参数复制到VTCompressionSessionEncodeFrame。
 * status:如果压缩成功则返回noErr;如果压缩不成功,则发出错误代码。
 * infoFlags:包含有关编码操作的信息。如果编码是异步运行的,则设置kVTEncodeInfo_Asynchronous。如果帧被丢弃,则设置kVTEncodeInfo_FrameDropped。
 * sampleBuffer:如果压缩成功且没有删除该帧,则包含该压缩帧;否则,空。
 */
typedef void (*VTCompressionOutputCallback)(
    void *outputCallbackRefCon, 
    void *sourceFrameRefCon, 
    OSStatus status, 
    VTEncodeInfoFlags infoFlags, 
    CMSampleBufferRef sampleBuffer
);

解码部分

初始化

- (void)decodeVideoInit {
    const uint8_t * const parameterSetPointers[2] = {_sps, _pps};
    const size_t parameterSetSizes[2] = {_spsSize, _ppsSize};
    int naluHeaderLen = 4;
    // 根据sps pps配置解码参数
    OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault, 2, parameterSetPointers, parameterSetSizes, naluHeaderLen, &_decodeDesc);
    if (status != noErr) {
        NSLog(@"Video hard DecodeSession create H264ParameterSets(sps, pps) failed status= %d", (int)status);
        return;
    }
    
    // 配置视频输出参数
    NSDictionary *destinationPixBufferAttrs =
    @{
      (id)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange], //iOS上 nv12(uvuv排布) 而不是nv21(vuvu排布)
      (id)kCVPixelBufferWidthKey: [NSNumber numberWithInteger:_width],
      (id)kCVPixelBufferHeightKey: [NSNumber numberWithInteger:_height],
      (id)kCVPixelBufferOpenGLCompatibilityKey: [NSNumber numberWithBool:true]
      };
    
    VTDecompressionOutputCallbackRecord callbackRecord;
    callbackRecord.decompressionOutputCallback = videoDecompressionOutputCallback;
    callbackRecord.decompressionOutputRefCon = (__bridge void * _Nullable)(self);
    
    // 创建解码器
    status = VTDecompressionSessionCreate(kCFAllocatorDefault, _decodeDesc, NULL, (__bridge CFDictionaryRef _Nullable)(destinationPixBufferAttrs), &callbackRecord, &_decoderSession);
    
    if (status != noErr) {
        NSLog(@"Video hard DecodeSession create failed status= %d", (int)status);
        return;
    }
    
    // 设置解码的实时
    VTSessionSetProperty(_decoderSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
}

CMVideoFormatDescriptionCreateFromH264ParameterSets参数解析

/*
 * allocator:分配器
 * parameterSetCount:参数个数
 * parameterSetPointers:参数集指针
 * parameterSetSizes:参数集大小
 * NALUnitHeaderLength:start code 的长度 4
 * formatDescriptionOut:解码器描述
 */
CM_EXPORT
OSStatus CMVideoFormatDescriptionCreateFromH264ParameterSets(
	CFAllocatorRef CM_NULLABLE allocator,								
	 size_t parameterSetCount,												
	 const uint8_t * CM_NONNULL const * CM_NONNULL parameterSetPointers,	
	 const size_t * CM_NONNULL parameterSetSizes,							
	 int NALUnitHeaderLength,											
	CM_RETURNS_RETAINED_PARAMETER CMFormatDescriptionRef CM_NULLABLE * CM_NONNULL formatDescriptionOut )

VTDecompressionOutputCallback 回调参数解析

/*
 * decompressionOutputRefCon:回调的引用
 * sourceFrameRefCon:帧的引用
 * status:一个状态标识 (包含未定义的代码)
 * infoFlags:指示同步/异步解码,或者解码器是否打算丢帧的标识
 * imageBuffer:实际图像的缓冲
 * presentationTimeStamp:出现的时间戳
 * presentationDuration:出现的持续时间
 */
typedef void (*VTDecompressionOutputCallback)(
		void * CM_NULLABLE decompressionOutputRefCon,
		void * CM_NULLABLE sourceFrameRefCon,
		OSStatus status, 
		VTDecodeInfoFlags infoFlags,
		CM_NULLABLE CVImageBufferRef imageBuffer,
		CMTime presentationTimeStamp, 
		CMTime presentationDuration );

VTDecompressionSessionCreate 参数解析

/*
 * allocator:内存的会话
 * videoFormatDescription:描述源视频帧
 * videoDecoderSpecification:指定必须使用的特定视频解码器
 * destinationImageBufferAttributes:描述源像素缓冲区的要求
 * outputCallback:使用已解压缩的帧调用的回调
 * decompressionSessionOut:指向一个变量以接收新的解压会话
 */
VT_EXPORT OSStatus 
VTDecompressionSessionCreate(
	CM_NULLABLE CFAllocatorRef                              allocator,
	CM_NONNULL CMVideoFormatDescriptionRef					videoFormatDescription,
	CM_NULLABLE CFDictionaryRef								videoDecoderSpecification,
	CM_NULLABLE CFDictionaryRef                             destinationImageBufferAttributes,
	const VTDecompressionOutputCallbackRecord * CM_NULLABLE outputCallback,
	CM_RETURNS_RETAINED_PARAMETER CM_NULLABLE VTDecompressionSessionRef * CM_NONNULL decompressionSessionOut) API_AVAILABLE(macosx(10.8), ios(8.0), tvos(10.2));

H264数据输入

- (void)decodeH264Data:(NSData *)h264Data {
    if (!self.decoderSession) {
        [self decodeVideoInit];
    }
    uint8_t *frame = (uint8_t *)h264Data.bytes;
    uint32_t size = (uint32_t)h264Data.length;
    int type = (frame[4] & 0x1F);
    
    // 将NALU的开始码转为4字节大端NALU的长度信息
    uint32_t naluSize = size - 4;
    uint8_t *pNaluSize = (uint8_t *)(&naluSize);
    frame[0] = *(pNaluSize + 3);
    frame[1] = *(pNaluSize + 2);
    frame[2] = *(pNaluSize + 1);
    frame[3] = *(pNaluSize);
    
    switch (type) {
        case 0x05: //关键帧
            [self decode:frame withSize:size];
            break;
        case 0x06:
            //NSLog(@"SEI");//增强信息
            break;
        case 0x07: //sps
            _spsSize = naluSize;
            _sps = malloc(_spsSize);
            memcpy(_sps, &frame[4], _spsSize);
            break;
        case 0x08: //pps
            _ppsSize = naluSize;
            _pps = malloc(_ppsSize);
            memcpy(_pps, &frame[4], _ppsSize);
            break;
        default: //其他帧(1-5)
            [self decode:frame withSize:size];
            break;
    }
    
}

// 解码函数
- (void)decode:(uint8_t *)frame withSize:(uint32_t)frameSize {
    CVPixelBufferRef outputPixelBuffer = NULL;
    CMBlockBufferRef blockBuffer = NULL;
    CMBlockBufferFlags flag0 = 0;
    
    OSStatus status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, frame, frameSize, kCFAllocatorNull, NULL, 0, frameSize, flag0, &blockBuffer);
    
    if (status != kCMBlockBufferNoErr) {
        NSLog(@"Video hard decode create blockBuffer error code=%d", (int)status);
        return;
    }
    
    CMSampleBufferRef sampleBuffer = NULL;
    const size_t sampleSizeArray[] = {frameSize};
    
    status = CMSampleBufferCreateReady(kCFAllocatorDefault, blockBuffer, _decodeDesc, 1, 0, NULL, 1, sampleSizeArray, &sampleBuffer);
    
    if (status != noErr || !sampleBuffer) {
        NSLog(@"Video hard decode create sampleBuffer failed status=%d", (int)status);
        CFRelease(blockBuffer);
        return;
    }
    
    //解码
    //向视频解码器提示使用低功耗模式是可以的
    VTDecodeFrameFlags flag1 = kVTDecodeFrame_1xRealTimePlayback;
    //异步解码
    VTDecodeInfoFlags  infoFlag = kVTDecodeInfo_Asynchronous;
    //解码数据
    /*
     参数1: 解码session
     参数2: 源数据 包含一个或多个视频帧的CMsampleBuffer
     参数3: 解码标志
     参数4: 解码后数据outputPixelBuffer
     参数5: 同步/异步解码标识
     */
    status = VTDecompressionSessionDecodeFrame(_decoderSession, sampleBuffer, flag1, &outputPixelBuffer, &infoFlag);
    
    if (status == kVTInvalidSessionErr) {
        NSLog(@"Video hard decode  InvalidSessionErr status =%d", (int)status);
    } else if (status == kVTVideoDecoderBadDataErr) {
        NSLog(@"Video hard decode  BadData status =%d", (int)status);
    } else if (status != noErr) {
        NSLog(@"Video hard decode failed status =%d", (int)status);
    }
    CFRelease(sampleBuffer);
    CFRelease(blockBuffer);
}

解码回调方法

void videoDecompressionOutputCallback(void * CM_NULLABLE decompressionOutputRefCon,
                                      void * CM_NULLABLE sourceFrameRefCon,
                                      OSStatus status,
                                      VTDecodeInfoFlags infoFlags,
                                      CM_NULLABLE CVImageBufferRef imageBuffer,
                                      CMTime presentationTimeStamp,
                                      CMTime presentationDuration ) {
    if (status != noErr) {
        NSLog(@"Video hard decode callback error status=%d", (int)status);
        return;
    }
    //解码后的数据sourceFrameRefCon -> CVPixelBufferRef
    CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *)sourceFrameRefCon;
    *outputPixelBuffer = CVPixelBufferRetain(imageBuffer);

    //获取self
    Demo2ViewController *decoder = (__bridge Demo2ViewController *)(decompressionOutputRefCon);
    
    //调用回调队列
    dispatch_async(decoder.decoderCallbackQueue, ^{
        
        // 这里异步把解码后的数据(imageBuffer)传输出去
        
        //释放数据
        CVPixelBufferRelease(imageBuffer);
    });
}

VTDecompressionOutputCallback 参数解析

/*
 * decompressionOutputRefCon:回调的引用
 * sourceFrameRefCon:帧的引用
 * status:一个状态标识 (包含未定义的代码)
 * infoFlags:指示同步/异步解码,或者解码器是否打算丢帧的标识
 * imageBuffer:实际图像的缓冲
 * presentationTimeStamp:出现的时间戳
 * presentationDuration:出现的持续时间
 */
typedef void (*VTDecompressionOutputCallback)(
		void * CM_NULLABLE decompressionOutputRefCon,
		void * CM_NULLABLE sourceFrameRefCon,
		OSStatus status, 
		VTDecodeInfoFlags infoFlags,
		CM_NULLABLE CVImageBufferRef imageBuffer,
		CMTime presentationTimeStamp, 
		CMTime presentationDuration );

完整代码

//
//  Demo2ViewController.m
//  Demo
//
//  Created by Noah on 2020/3/19.
//  Copyright © 2020 Noah. All rights reserved.
//

#import "Demo2ViewController.h"
#import <AVFoundation/AVFoundation.h>
#import <VideoToolbox/VideoToolbox.h>

@interface Demo2ViewController ()<AVCaptureVideoDataOutputSampleBufferDelegate>
{
    long frameID;
    BOOL hasSpsPps;//判断是否已经获取到pps和sps
    
    NSMutableData *sps;
    NSMutableData *pps;
    
    
    uint8_t *_sps;
    NSUInteger _spsSize;
    uint8_t *_pps;
    NSUInteger _ppsSize;
    CMVideoFormatDescriptionRef _decodeDesc;
}



@property (nonatomic, strong) AVCaptureSession *captureSession;
@property (nonatomic, strong) dispatch_queue_t captureQueue;

@property (nonatomic, strong) AVCaptureDeviceInput *videoDataInput;
@property (nonatomic, strong) AVCaptureDeviceInput *frontCamera;
@property (nonatomic, strong) AVCaptureDeviceInput *backCamera;

@property (nonatomic, strong) AVCaptureVideoDataOutput *videoDataOutput;
@property (nonatomic, strong) AVCaptureConnection *videoConnection;

@property (nonatomic, strong) AVCaptureVideoPreviewLayer *videoPreviewLayer;

@property (nonatomic) VTCompressionSessionRef encoderSession;
@property (nonatomic, strong) dispatch_queue_t encoderQueue;
@property (nonatomic, strong) dispatch_queue_t encoderCallbackQueue;

@property (nonatomic) VTDecompressionSessionRef decoderSession;
@property (nonatomic, strong) dispatch_queue_t decoderQueue;
@property (nonatomic, strong) dispatch_queue_t decoderCallbackQueue;

// 捕获视频的宽
@property (nonatomic, assign, readonly) NSUInteger width;
// 捕获视频的高
@property (nonatomic, assign, readonly) NSUInteger height;

@end

const Byte StartCode[] = "\x00\x00\x00\x01";

@implementation Demo2ViewController

- (void)viewDidLoad {
    [super viewDidLoad];
    // Do any additional setup after loading the view.
    [self setupVideo];
    
    [self.captureSession startRunning];
}

#pragma mark - 视频初始化
- (void)setupVideo{
    NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
    for (AVCaptureDevice *device in devices) {
        if (device.position == AVCaptureDevicePositionBack) {
            self.backCamera = [AVCaptureDeviceInput deviceInputWithDevice:device error:nil];
        }else{
            self.frontCamera = [AVCaptureDeviceInput deviceInputWithDevice:device error:nil];
        }
    }
    
    self.videoDataInput = self.backCamera;
    
    self.videoDataOutput = [[AVCaptureVideoDataOutput alloc]init];
    [self.videoDataOutput setSampleBufferDelegate:self queue:self.captureQueue];
    [self.videoDataOutput setAlwaysDiscardsLateVideoFrames:YES];
    [self.videoDataOutput setVideoSettings:@{(__bridge NSString *)kCVPixelBufferPixelFormatTypeKey:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)}];
    
    [self.captureSession beginConfiguration];
    if ([self.captureSession canAddInput:self.videoDataInput]) {
        [self.captureSession addInput:self.videoDataInput];
    }
    if ([self.captureSession canAddOutput:self.videoDataOutput]) {
        [self.captureSession addOutput:self.videoDataOutput];
    }
    [self setVideoPreset];
    [self.captureSession commitConfiguration];
    
    self.videoConnection = [self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
    self.videoConnection.videoOrientation = AVCaptureVideoOrientationPortrait;
    
    [self updateFps:25];
    
    self.videoPreviewLayer = [AVCaptureVideoPreviewLayer layerWithSession:self.captureSession];
    self.videoPreviewLayer.frame = self.view.bounds;
    self.videoPreviewLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
    [self.view.layer addSublayer:self.videoPreviewLayer];
    
}

- (void)setVideoPreset {
    if ([self.captureSession canSetSessionPreset:AVCaptureSessionPreset1920x1080]) {
        [self.captureSession setSessionPreset:AVCaptureSessionPreset1920x1080];
        _width = 1080; _height = 1920;
    } else if ([self.captureSession canSetSessionPreset:AVCaptureSessionPreset1280x720]) {
        [self.captureSession setSessionPreset:AVCaptureSessionPreset1280x720];
        _width = 720; _height = 1280;
    } else if ([self.captureSession canSetSessionPreset:AVCaptureSessionPreset640x480]) {
        [self.captureSession setSessionPreset:AVCaptureSessionPreset640x480];
        _width = 480; _height = 640;
    }
}

-(void)updateFps:(NSInteger) fps{
    //获取当前capture设备
    NSArray *videoDevices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
    
    //遍历所有设备(前后摄像头)
    for (AVCaptureDevice *vDevice in videoDevices) {
        //获取当前支持的最大fps
        float maxRate = [(AVFrameRateRange *)[vDevice.activeFormat.videoSupportedFrameRateRanges objectAtIndex:0] maxFrameRate];
        //如果想要设置的fps小于或等于做大fps,就进行修改
        if (maxRate >= fps) {
            //实际修改fps的代码
            if ([vDevice lockForConfiguration:NULL]) {
                vDevice.activeVideoMinFrameDuration = CMTimeMake(10, (int)(fps * 10));
                vDevice.activeVideoMaxFrameDuration = vDevice.activeVideoMinFrameDuration;
                [vDevice unlockForConfiguration];
            }
        }
    }
}

#pragma mark - 懒加载
- (AVCaptureSession *)captureSession{
    if (!_captureSession) {
        _captureSession = [[AVCaptureSession alloc]init];
    }
    return _captureSession;
}

- (dispatch_queue_t)captureQueue{
    if (!_captureQueue) {
        _captureQueue = dispatch_queue_create("capture queue", NULL);
    }
    return _captureQueue;
}

- (dispatch_queue_t)encoderQueue{
    if (!_captureQueue) {
        _captureQueue = dispatch_queue_create("encoder queue", NULL);
    }
    return _captureQueue;
}

- (dispatch_queue_t)encoderCallbackQueue{
    if (!_encoderCallbackQueue) {
        _encoderCallbackQueue = dispatch_queue_create("encoder callback queue", NULL);
    }
    return _encoderCallbackQueue;
}

- (dispatch_queue_t)decoderQueue{
    if (!_decoderQueue) {
        _decoderQueue = dispatch_queue_create("decoder queue", NULL);
    }
    return _decoderQueue;
}

- (dispatch_queue_t)decoderCallbackQueue{
    if (!_decoderCallbackQueue) {
        _decoderCallbackQueue = dispatch_queue_create("decoder callback queue", NULL);
    }
    return _decoderCallbackQueue;
}

#pragma mark - 编码
- (void)encodeInit{
    OSStatus status = VTCompressionSessionCreate(kCFAllocatorDefault, (uint32_t)_width, (uint32_t)_height, kCMVideoCodecType_H264, NULL, NULL, NULL, encodeCallBack, (__bridge void * _Nullable)(self), &_encoderSession);
    if (status != noErr) {
        return;
    }
    // 设置实时编码
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
    // 设置不需要B帧
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
    // 设置码率均值
    CFNumberRef bitRate = (__bridge CFNumberRef)(@(_height*1000));
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_AverageBitRate, bitRate);
    // 设置最值码率
    CFArrayRef limits = (__bridge CFArrayRef)(@[@(_height*1000/4),@(_height*1000*4)]);
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_DataRateLimits, limits);
    // 设置FPS
    CFNumberRef fps = (__bridge CFNumberRef)(@(25));
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_ExpectedFrameRate, fps);
    // 设置I帧间隔
    CFNumberRef maxKeyFrameInterval = (__bridge CFNumberRef)(@(25*2));
    status = VTSessionSetProperty(_encoderSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, maxKeyFrameInterval);
    // 准备编码
    status = VTCompressionSessionPrepareToEncodeFrames(_encoderSession);
}

void encodeCallBack (
void * CM_NULLABLE outputCallbackRefCon,
void * CM_NULLABLE sourceFrameRefCon,
OSStatus status,
VTEncodeInfoFlags infoFlags,
CM_NULLABLE CMSampleBufferRef sampleBuffer ){
    if (status != noErr) {
        NSLog(@"encodeVideoCallBack: encode error, status = %d",(int)status);
        return;
    }
    
    if (!CMSampleBufferDataIsReady(sampleBuffer)) {
        NSLog(@"encodeVideoCallBack: data is not ready");
        return;
    }
    
    Demo2ViewController *VC = (__bridge Demo2ViewController *)(outputCallbackRefCon);
    
    // 判断是否为关键帧
    BOOL isKeyFrame = NO;
    CFArrayRef attachArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
    isKeyFrame = !CFDictionaryContainsKey(CFArrayGetValueAtIndex(attachArray, 0), kCMSampleAttachmentKey_NotSync);//(注意取反符号)
    
    if (isKeyFrame && !VC->hasSpsPps) {
        size_t spsSize, spsCount;
        size_t ppsSize, ppsCount;
        const uint8_t *spsData, *ppsData;
        
        // 获取图像源格式
        CMFormatDescriptionRef formatDesc = CMSampleBufferGetFormatDescription(sampleBuffer);
        OSStatus status1 = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(formatDesc, 0, &spsData, &spsSize, &spsCount, 0);
        OSStatus status2 = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(formatDesc, 1, &ppsData, &ppsSize, &ppsCount, 0);
        
        if (status1 == noErr && status2 == noErr) {
            VC->hasSpsPps = true;
            
            //sps data
            VC->sps = [NSMutableData dataWithCapacity:4 + spsSize];
            [VC->sps appendBytes:StartCode length:4];
            [VC->sps appendBytes:spsData length:spsSize];
            //pps data
            VC->pps = [NSMutableData dataWithCapacity:4 + ppsSize];
            [VC->pps appendBytes:StartCode length:4];
            [VC->pps appendBytes:ppsData length:ppsSize];
            
            [VC decodeH264Data:VC->sps];
            [VC decodeH264Data:VC->pps];
        }
    }
    
    // 获取NALU数据
    size_t lengthAtOffset, totalLength;
    char *dataPoint;
    
    // 将数据复制到dataPoint
    CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
    OSStatus error = CMBlockBufferGetDataPointer(blockBuffer, 0, &lengthAtOffset, &totalLength, &dataPoint);
    if (error != kCMBlockBufferNoErr) {
        NSLog(@"VideoEncodeCallback: get datapoint failed, status = %d", (int)error);
        return;
    }
    
    // 循环获取nalu数据
    size_t offet = 0;
    // 返回的nalu数据前四个字节不是0001的startcode(不是系统端的0001),而是大端模式的帧长度length
    const int lengthInfoSize = 4;
    while (offet < totalLength - lengthInfoSize) {
        uint32_t naluLength = 0;
        // 获取nalu 数据长度
        memcpy(&naluLength, dataPoint + offet, lengthInfoSize);
        // 大端转系统端
        naluLength = CFSwapInt32BigToHost(naluLength);
        // 获取到编码好的视频数据
        NSMutableData *data = [NSMutableData dataWithCapacity:4 + naluLength];
        [data appendBytes:StartCode length:4];
        [data appendBytes:dataPoint + offet + lengthInfoSize length:naluLength];
        
        dispatch_async(VC.encoderCallbackQueue, ^{
            [VC decodeH264Data:data];
        });
        
        // 移动下标,继续读取下一个数据
        offet += lengthInfoSize + naluLength;
    }
}

- (void)encodeSampleBuffer:(CMSampleBufferRef)sampleBuffer{
    
    if (!self.encoderSession) {
        [self encodeInit];
    }
    
    CFRetain(sampleBuffer);
    dispatch_async(self.encoderQueue, ^{
        CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
        CMTime TimeStamp = CMTimeMake(self->frameID, 1000);
        self->frameID++;
        CMTime duration = kCMTimeInvalid;
        VTEncodeInfoFlags infoFlagsOut;
        OSStatus status = VTCompressionSessionEncodeFrame(self.encoderSession, imageBuffer, TimeStamp, duration, NULL, NULL, &infoFlagsOut);
        if (status != noErr) {
            NSLog(@"error");
        }
        CFRelease(sampleBuffer);
    });
}

#pragma mark - 视频解码
// 视频解码器初始化
- (void)decodeVideoInit {
    const uint8_t * const parameterSetPointers[2] = {_sps, _pps};
    const size_t parameterSetSizes[2] = {_spsSize, _ppsSize};
    int naluHeaderLen = 4;
    // 根据sps pps配置解码参数
    OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault, 2, parameterSetPointers, parameterSetSizes, naluHeaderLen, &_decodeDesc);
    if (status != noErr) {
        NSLog(@"Video hard DecodeSession create H264ParameterSets(sps, pps) failed status= %d", (int)status);
        return;
    }
    
    // 配置视频输出参数
    NSDictionary *destinationPixBufferAttrs =
    @{
      (id)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange], //iOS上 nv12(uvuv排布) 而不是nv21(vuvu排布)
      (id)kCVPixelBufferWidthKey: [NSNumber numberWithInteger:_width],
      (id)kCVPixelBufferHeightKey: [NSNumber numberWithInteger:_height],
      (id)kCVPixelBufferOpenGLCompatibilityKey: [NSNumber numberWithBool:true]
      };
    
    VTDecompressionOutputCallbackRecord callbackRecord;
    callbackRecord.decompressionOutputCallback = videoDecompressionOutputCallback;
    callbackRecord.decompressionOutputRefCon = (__bridge void * _Nullable)(self);
    
    // 创建解码器
    status = VTDecompressionSessionCreate(kCFAllocatorDefault, _decodeDesc, NULL, (__bridge CFDictionaryRef _Nullable)(destinationPixBufferAttrs), &callbackRecord, &_decoderSession);
    
    if (status != noErr) {
        NSLog(@"Video hard DecodeSession create failed status= %d", (int)status);
        return;
    }
    
    // 设置解码的实时
    VTSessionSetProperty(_decoderSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
    
}

/**解码回调函数*/
void videoDecompressionOutputCallback(void * CM_NULLABLE decompressionOutputRefCon,
                                      void * CM_NULLABLE sourceFrameRefCon,
                                      OSStatus status,
                                      VTDecodeInfoFlags infoFlags,
                                      CM_NULLABLE CVImageBufferRef imageBuffer,
                                      CMTime presentationTimeStamp,
                                      CMTime presentationDuration ) {
    if (status != noErr) {
        NSLog(@"Video hard decode callback error status=%d", (int)status);
        return;
    }
    //解码后的数据sourceFrameRefCon -> CVPixelBufferRef
    CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *)sourceFrameRefCon;
    *outputPixelBuffer = CVPixelBufferRetain(imageBuffer);

    //获取self
    Demo2ViewController *decoder = (__bridge Demo2ViewController *)(decompressionOutputRefCon);
    
    //调用回调队列
    dispatch_async(decoder.decoderCallbackQueue, ^{
        
        // 这里异步把解码后的数据(imageBuffer)传输出去
        
        //释放数据
        CVPixelBufferRelease(imageBuffer);
    });
}

// 解码
- (void)decodeH264Data:(NSData *)h264Data {
    if (!self.decoderSession) {
        [self decodeVideoInit];
    }
    uint8_t *frame = (uint8_t *)h264Data.bytes;
    uint32_t size = (uint32_t)h264Data.length;
    int type = (frame[4] & 0x1F);
    
    // 将NALU的开始码转为4字节大端NALU的长度信息
    uint32_t naluSize = size - 4;
    uint8_t *pNaluSize = (uint8_t *)(&naluSize);
    frame[0] = *(pNaluSize + 3);
    frame[1] = *(pNaluSize + 2);
    frame[2] = *(pNaluSize + 1);
    frame[3] = *(pNaluSize);
    
    switch (type) {
        case 0x05: //关键帧
            [self decode:frame withSize:size];
            break;
        case 0x06:
            //NSLog(@"SEI");//增强信息
            break;
        case 0x07: //sps
            _spsSize = naluSize;
            _sps = malloc(_spsSize);
            memcpy(_sps, &frame[4], _spsSize);
            break;
        case 0x08: //pps
            _ppsSize = naluSize;
            _pps = malloc(_ppsSize);
            memcpy(_pps, &frame[4], _ppsSize);
            break;
        default: //其他帧(1-5)
            [self decode:frame withSize:size];
            break;
    }
    
}

// 解码函数
- (void)decode:(uint8_t *)frame withSize:(uint32_t)frameSize {
    CVPixelBufferRef outputPixelBuffer = NULL;
    CMBlockBufferRef blockBuffer = NULL;
    CMBlockBufferFlags flag0 = 0;
    
    OSStatus status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, frame, frameSize, kCFAllocatorNull, NULL, 0, frameSize, flag0, &blockBuffer);
    
    if (status != kCMBlockBufferNoErr) {
        NSLog(@"Video hard decode create blockBuffer error code=%d", (int)status);
        return;
    }
    
    CMSampleBufferRef sampleBuffer = NULL;
    const size_t sampleSizeArray[] = {frameSize};
    
    status = CMSampleBufferCreateReady(kCFAllocatorDefault, blockBuffer, _decodeDesc, 1, 0, NULL, 1, sampleSizeArray, &sampleBuffer);
    
    if (status != noErr || !sampleBuffer) {
        NSLog(@"Video hard decode create sampleBuffer failed status=%d", (int)status);
        CFRelease(blockBuffer);
        return;
    }
    
    //解码
    //向视频解码器提示使用低功耗模式是可以的
    VTDecodeFrameFlags flag1 = kVTDecodeFrame_1xRealTimePlayback;
    //异步解码
    VTDecodeInfoFlags  infoFlag = kVTDecodeInfo_Asynchronous;
    //解码数据
    /*
     参数1: 解码session
     参数2: 源数据 包含一个或多个视频帧的CMsampleBuffer
     参数3: 解码标志
     参数4: 解码后数据outputPixelBuffer
     参数5: 同步/异步解码标识
     */
    status = VTDecompressionSessionDecodeFrame(_decoderSession, sampleBuffer, flag1, &outputPixelBuffer, &infoFlag);
    
    if (status == kVTInvalidSessionErr) {
        NSLog(@"Video hard decode  InvalidSessionErr status =%d", (int)status);
    } else if (status == kVTVideoDecoderBadDataErr) {
        NSLog(@"Video hard decode  BadData status =%d", (int)status);
    } else if (status != noErr) {
        NSLog(@"Video hard decode failed status =%d", (int)status);
    }
    CFRelease(sampleBuffer);
    CFRelease(blockBuffer);
}


#pragma mark - AVCaptureVideoDataOutputSampleBufferDelegate
- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection{
    if (connection == self.videoConnection) {
        [self encodeSampleBuffer:sampleBuffer];
    }
}

#pragma mark - 析构
- (void)dealloc {
    if (_decoderSession) {
        VTDecompressionSessionInvalidate(_decoderSession);
        CFRelease(_decoderSession);
        _decoderSession = NULL;
    }
    if (_encoderSession) {
        VTCompressionSessionInvalidate(_encoderSession);
        CFRelease(_encoderSession);
        _encoderSession = NULL;
    }
}

@end

AudioToolbox

编码部分

初始化

- (void)setupAudioConverter:(CMSampleBufferRef)sampleBuffer{
    // 通过实时的解码数据来获取输入的信息
    AudioStreamBasicDescription inputDescription = *CMAudioFormatDescriptionGetStreamBasicDescription(CMSampleBufferGetFormatDescription(sampleBuffer));
    // 定义pcm输出信息
    AudioStreamBasicDescription outputDescription =
    {
        .mSampleRate = 44100,
        .mFormatID = kAudioFormatMPEG4AAC,
        .mFormatFlags = kMPEG4Object_AAC_LC,
        .mBytesPerPacket = 0,
        .mFramesPerPacket = 1024,
        .mBytesPerFrame = 0,
        .mChannelsPerFrame = 1,
        .mBitsPerChannel = 0,
        .mReserved = 0
    };
    // 创建解码器
    OSStatus status = AudioConverterNew(&inputDescription, &outputDescription, &_audioConverter);
    if (status != noErr) {
        NSLog(@"error");
        return;
    }
    // 设置输出质量
    UInt32 temp = kAudioConverterQuality_High;
    AudioConverterSetProperty(_audioConverter, kAudioConverterCodecQuality, sizeof(temp), &temp);
    // 设置比特率
    UInt32 bitRate = 96000;
    AudioConverterSetProperty(_audioConverter, kAudioConverterEncodeBitRate, sizeof(bitRate), &bitRate);
}

首先通过实时的音频数据获取输入的信息,再定义pcm输出信息,然后通过这两个参数去创建解码器。创建成功之后设置一些解码器的属性。

编码函数

- (void)setAudioSampleBuffer:(CMSampleBufferRef)sampleBuffer{
    CFRetain(sampleBuffer);
    if (!self.audioConverter) {
        [self setupAudioConverter:sampleBuffer];
    }
    
    dispatch_async(self.encodeQueue, ^{
        CMBlockBufferRef bufferRef = CMSampleBufferGetDataBuffer(sampleBuffer);
        CFRetain(bufferRef);
        CMBlockBufferGetDataPointer(bufferRef, 0, NULL, &self->_pcmBufferSize, &self->_pcmBuffer);
        
        char *pcmbuffer = malloc(self->_pcmBufferSize);
        memset(pcmbuffer, 0, self->_pcmBufferSize);
        
        AudioBufferList audioBufferList = {0};
        audioBufferList.mNumberBuffers = 1;
        audioBufferList.mBuffers[0].mData = pcmbuffer;
        audioBufferList.mBuffers[0].mDataByteSize = (uint32_t)self->_pcmBufferSize;
        audioBufferList.mBuffers[0].mNumberChannels = 1;
        UInt32 dataPacketSize = 1;
        
        OSStatus status = AudioConverterFillComplexBuffer(self.audioConverter, aacEncodeInputDataProc, (__bridge void * _Nullable)(self), &dataPacketSize, &audioBufferList, NULL);
        if (status == noErr) {
            NSData *aacData = [NSData dataWithBytes:audioBufferList.mBuffers[0].mData length:audioBufferList.mBuffers[0].mDataByteSize];
            free(pcmbuffer);
            //添加ADTS头,想要获取裸流时,请忽略添加ADTS头,写入文件时,必须添加
            //NSData *adtsHeader = [self adtsDataForPacketLength:rawAAC.length];
            //NSMutableData *fullData = [NSMutableData dataWithCapacity:adtsHeader.length + rawAAC.length];
            //[fullData appendData:adtsHeader];
            //[fullData appendData:rawAAC];
            dispatch_async(self.encodeCallbackQueue, ^{
                [self decodeAudioAACData:aacData];
            });
        }
        CFRelease(bufferRef);
        CFRelease(sampleBuffer);
    });
}

音频解码和视频解码不一样,首先要获取出pcm的数据存放在一个全局变量里面,然后创建一个AudioBufferList用于接收数据,解码工作在解码回调里面。把回调的指针以及创建的AudioBufferList传入AudioConverterFillComplexBuffer中,这样就会不断的解码数据了。

编码回调

static OSStatus aacEncodeInputDataProc(
AudioConverterRef inAudioConverter,
UInt32 *ioNumberDataPackets,
AudioBufferList *ioData,
AudioStreamPacketDescription **outDataPacketDescription,
void *inUserData){
    AudioViewController *audioVC = (__bridge AudioViewController *)(inUserData);
    
    if (!audioVC.pcmBufferSize) {
        *ioNumberDataPackets = 0;
        return -1;
    }
    // 填充数据
    ioData->mBuffers[0].mDataByteSize = (UInt32)audioVC.pcmBufferSize;
    ioData->mBuffers[0].mData = audioVC.pcmBuffer;
    ioData->mBuffers[0].mNumberChannels = 1;
    
    audioVC.pcmBufferSize = 0;
    *ioNumberDataPackets = 1;
    return noErr;
}

编码回调函数里面做了填充PCM数据的工作,这样就可以进行编码

解码部分

初始化

- (void)setupEncoder {
    //输出参数pcm
    AudioStreamBasicDescription outputAudioDes = {0};
    outputAudioDes.mSampleRate = (Float64)self.sampleRate;       //采样率
    outputAudioDes.mChannelsPerFrame = (UInt32)self.channelCount; //输出声道数
    outputAudioDes.mFormatID = kAudioFormatLinearPCM;                //输出格式
    outputAudioDes.mFormatFlags = (kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked); //编码 12
    outputAudioDes.mFramesPerPacket = 1;                            //每一个packet帧数 ;
    outputAudioDes.mBitsPerChannel = 16;                             //数据帧中每个通道的采样位数。
    outputAudioDes.mBytesPerFrame = outputAudioDes.mBitsPerChannel / 8 *outputAudioDes.mChannelsPerFrame;                              //每一帧大小(采样位数 / 8 *声道数)
    outputAudioDes.mBytesPerPacket = outputAudioDes.mBytesPerFrame * outputAudioDes.mFramesPerPacket;                             //每个packet大小(帧大小 * 帧数)
    outputAudioDes.mReserved =  0;                                  //对其方式 0(8字节对齐)
    
    //输入参数aac
    AudioStreamBasicDescription inputAduioDes = {0};
    inputAduioDes.mSampleRate = (Float64)self.sampleRate;
    inputAduioDes.mFormatID = kAudioFormatMPEG4AAC;
    inputAduioDes.mFormatFlags = kMPEG4Object_AAC_LC;
    inputAduioDes.mFramesPerPacket = 1024;
    inputAduioDes.mChannelsPerFrame = (UInt32)self.channelCount;
    
    //填充输入相关信息
    UInt32 inDesSize = sizeof(inputAduioDes);
    AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, NULL, &inDesSize, &inputAduioDes);
    
    OSStatus status = AudioConverterNew(&inputAduioDes, &outputAudioDes, &_audioDecodeConverter);
    if (status != noErr) {
        NSLog(@"Error!:硬解码AAC创建失败, status= %d", (int)status);
        return;
    }
}

解码器的初始化与编码器的初始化类似,不过多阐述

解码函数

- (void)decodeAudioAACData:(NSData *)aacData {
   
    if (!_audioDecodeConverter) {
        [self setupEncoder];
    }
    
    dispatch_async(self.decodeQueue, ^{
     
        //记录aac 作为参数参入解码回调函数
        CCAudioUserData userData = {0};
        userData.channelCount = (UInt32)self.channelCount;
        userData.data = (char *)[aacData bytes];
        userData.size = (UInt32)aacData.length;
        userData.packetDesc.mDataByteSize = (UInt32)aacData.length;
        userData.packetDesc.mStartOffset = 0;
        userData.packetDesc.mVariableFramesInPacket = 0;
        
        //输出大小和packet个数
        UInt32 pcmBufferSize = (UInt32)(2048 * self.channelCount);
        UInt32 pcmDataPacketSize = 1024;
        
        //创建临时容器pcm
        uint8_t *pcmBuffer = malloc(pcmBufferSize);
        memset(pcmBuffer, 0, pcmBufferSize);
        
        //输出buffer
        AudioBufferList outAudioBufferList = {0};
        outAudioBufferList.mNumberBuffers = 1;
        outAudioBufferList.mBuffers[0].mNumberChannels = (uint32_t)self.channelCount;
        outAudioBufferList.mBuffers[0].mDataByteSize = (UInt32)pcmBufferSize;
        outAudioBufferList.mBuffers[0].mData = pcmBuffer;
        
        //输出描述
        AudioStreamPacketDescription outputPacketDesc = {0};
        
        //配置填充函数,获取输出数据
        OSStatus status = AudioConverterFillComplexBuffer(self->_audioDecodeConverter, AudioDecoderConverterComplexInputDataProc, &userData, &pcmDataPacketSize, &outAudioBufferList, &outputPacketDesc);
        if (status != noErr) {
            NSLog(@"Error: AAC Decoder error, status=%d",(int)status);
            return;
        }
        //如果获取到数据
        if (outAudioBufferList.mBuffers[0].mDataByteSize > 0) {
            NSData *rawData = [NSData dataWithBytes:outAudioBufferList.mBuffers[0].mData length:outAudioBufferList.mBuffers[0].mDataByteSize];
            dispatch_async(self.decodeCallbackQueue, ^{
                // 这里可以处理解码出来的pcm数据
                NSLog(@"%@",rawData);
            });
        }
        free(pcmBuffer);
    });
    
}

音频的解码函数和编码函数都是AudioConverterFillComplexBuffer,参数也是一样,唯一的不同是解码后输出的数据是PCM数据

解码回调

static OSStatus AudioDecoderConverterComplexInputDataProc(  AudioConverterRef inAudioConverter, UInt32 *ioNumberDataPackets, AudioBufferList *ioData,  AudioStreamPacketDescription **outDataPacketDescription,  void *inUserData) {
    
    
    CCAudioUserData *audioDecoder = (CCAudioUserData *)(inUserData);
    if (audioDecoder->size <= 0) {
        ioNumberDataPackets = 0;
        return -1;
    }
   
    //填充数据
    *outDataPacketDescription = &audioDecoder->packetDesc;
    (*outDataPacketDescription)[0].mStartOffset = 0;
    (*outDataPacketDescription)[0].mDataByteSize = audioDecoder->size;
    (*outDataPacketDescription)[0].mVariableFramesInPacket = 0;
    
    ioData->mBuffers[0].mData = audioDecoder->data;
    ioData->mBuffers[0].mDataByteSize = audioDecoder->size;
    ioData->mBuffers[0].mNumberChannels = audioDecoder->channelCount;
    
    return noErr;
}

完整代码

//
//  AudioViewController.m
//  Demo
//
//  Created by Noah on 2020/3/21.
//  Copyright © 2020 Noah. All rights reserved.
//

#import "AudioViewController.h"
#import <AVFoundation/AVFoundation.h>
#import <AudioToolbox/AudioToolbox.h>

typedef struct {
    char * data;
    UInt32 size;
    UInt32 channelCount;
    AudioStreamPacketDescription packetDesc;
} CCAudioUserData;

@interface AudioViewController ()<AVCaptureAudioDataOutputSampleBufferDelegate>

@property (nonatomic, strong) AVCaptureSession *captureSession;
@property (nonatomic, strong) dispatch_queue_t captureQueue;

@property (nonatomic, strong) AVCaptureInput *audioDeviceInput;
@property (nonatomic, strong) AVCaptureAudioDataOutput *audioDeviceOutput;

@property (nonatomic, strong) dispatch_queue_t encodeQueue;
@property (nonatomic, strong) dispatch_queue_t encodeCallbackQueue;

@property (nonatomic, strong) AVCaptureConnection *audioConnettion;
@property (nonatomic, unsafe_unretained) AudioConverterRef audioConverter;

@property (nonatomic) char *pcmBuffer;
@property (nonatomic) size_t pcmBufferSize;

@property (nonatomic, strong) dispatch_queue_t decodeQueue;
@property (nonatomic, strong) dispatch_queue_t decodeCallbackQueue;

@property (nonatomic) AudioConverterRef audioDecodeConverter;
@property (nonatomic) char *aacBuffer;
@property (nonatomic) UInt32 aacBufferSize;

/**码率*/
@property (nonatomic, assign) NSInteger bitrate;//(96000)
/**声道*/
@property (nonatomic, assign) NSInteger channelCount;//(1)
/**采样率*/
@property (nonatomic, assign) NSInteger sampleRate;//(默认44100)
/**采样点量化*/
@property (nonatomic, assign) NSInteger sampleSize;//(16)


@end

@implementation AudioViewController

- (void)viewDidLoad {
    [super viewDidLoad];
    // Do any additional setup after loading the view.
    
    self.bitrate = 96000;
    self.channelCount = 1;
    self.sampleRate = 44100;
    self.sampleSize = 16;
    
    [self setupAudio];
    [self.captureSession startRunning];
    
    
}

#pragma mark - 懒加载
- (AVCaptureSession *)captureSession{
    if (!_captureSession) {
        _captureSession = [[AVCaptureSession alloc]init];
    }
    return _captureSession;
}

- (dispatch_queue_t)captureQueue{
    if (!_captureQueue) {
        _captureQueue = dispatch_queue_create("capture queue", NULL);
    }
    return _captureQueue;
}

- (dispatch_queue_t)encodeQueue{
    if (!_encodeQueue) {
        _encodeQueue = dispatch_queue_create("encode queue", NULL);
    }
    return _encodeQueue;
}

- (dispatch_queue_t)encodeCallbackQueue{
    if (!_encodeCallbackQueue) {
        _encodeCallbackQueue = dispatch_queue_create("encode callback queue", NULL);
    }
    return _encodeCallbackQueue;
}

- (dispatch_queue_t)decodeQueue{
    if (!_decodeQueue) {
        _decodeQueue = dispatch_queue_create("decode queue", NULL);
    }
    return _decodeQueue;
}

- (dispatch_queue_t)decodeCallbackQueue{
    if (!_decodeCallbackQueue) {
        _decodeCallbackQueue = dispatch_queue_create("decode callback queue", NULL);
    }
    return _decodeCallbackQueue;
}

#pragma mark - 音频初始化
- (void)setupAudio{
    AVCaptureDevice *audioDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
    
    self.audioDeviceInput = [AVCaptureDeviceInput deviceInputWithDevice:audioDevice error:nil];
    
    self.audioDeviceOutput = [[AVCaptureAudioDataOutput alloc]init];
    [self.audioDeviceOutput setSampleBufferDelegate:self queue:self.captureQueue];
    
    [self.captureSession beginConfiguration];
    if ([self.captureSession canAddInput:self.audioDeviceInput]) {
        [self.captureSession addInput:self.audioDeviceInput];
    }
    if ([self.captureSession canAddOutput:self.audioDeviceOutput]) {
        [self.captureSession addOutput:self.audioDeviceOutput];
    }
    [self.captureSession commitConfiguration];
    
    self.audioConnettion = [self.audioDeviceOutput connectionWithMediaType:AVMediaTypeAudio];
    
}

#pragma mark - 音频编码
- (void)setupAudioConverter:(CMSampleBufferRef)sampleBuffer{
    AudioStreamBasicDescription inputDescription = *CMAudioFormatDescriptionGetStreamBasicDescription(CMSampleBufferGetFormatDescription(sampleBuffer));
    
    AudioStreamBasicDescription outputDescription =
    {
        .mSampleRate = 44100,
        .mFormatID = kAudioFormatMPEG4AAC,
        .mFormatFlags = kMPEG4Object_AAC_LC,
        .mBytesPerPacket = 0,
        .mFramesPerPacket = 1024,
        .mBytesPerFrame = 0,
        .mChannelsPerFrame = 1,
        .mBitsPerChannel = 0,
        .mReserved = 0
    };
    
    OSStatus status = AudioConverterNew(&inputDescription, &outputDescription, &_audioConverter);
    if (status != noErr) {
        NSLog(@"error");
        return;
    }
    UInt32 temp = kAudioConverterQuality_High;
    AudioConverterSetProperty(_audioConverter, kAudioConverterCodecQuality, sizeof(temp), &temp);
    
    UInt32 bitRate = 96000;
    AudioConverterSetProperty(_audioConverter, kAudioConverterEncodeBitRate, sizeof(bitRate), &bitRate);
}

static OSStatus aacEncodeInputDataProc(
AudioConverterRef inAudioConverter,
UInt32 *ioNumberDataPackets,
AudioBufferList *ioData,
AudioStreamPacketDescription **outDataPacketDescription,
void *inUserData){
    AudioViewController *audioVC = (__bridge AudioViewController *)(inUserData);
    
    if (!audioVC.pcmBufferSize) {
        *ioNumberDataPackets = 0;
        return -1;
    }
    
    ioData->mBuffers[0].mDataByteSize = (UInt32)audioVC.pcmBufferSize;
    ioData->mBuffers[0].mData = audioVC.pcmBuffer;
    ioData->mBuffers[0].mNumberChannels = 1;
    
    audioVC.pcmBufferSize = 0;
    *ioNumberDataPackets = 1;
    return noErr;
}

- (void)setAudioSampleBuffer:(CMSampleBufferRef)sampleBuffer{
    CFRetain(sampleBuffer);
    if (!self.audioConverter) {
        [self setupAudioConverter:sampleBuffer];
    }
    
    dispatch_async(self.encodeQueue, ^{
        CMBlockBufferRef bufferRef = CMSampleBufferGetDataBuffer(sampleBuffer);
        CFRetain(bufferRef);
        CMBlockBufferGetDataPointer(bufferRef, 0, NULL, &self->_pcmBufferSize, &self->_pcmBuffer);
        
        char *pcmbuffer = malloc(self->_pcmBufferSize);
        memset(pcmbuffer, 0, self->_pcmBufferSize);
        
        AudioBufferList audioBufferList = {0};
        audioBufferList.mNumberBuffers = 1;
        audioBufferList.mBuffers[0].mData = pcmbuffer;
        audioBufferList.mBuffers[0].mDataByteSize = (uint32_t)self->_pcmBufferSize;
        audioBufferList.mBuffers[0].mNumberChannels = 1;
        UInt32 dataPacketSize = 1;
        
        OSStatus status = AudioConverterFillComplexBuffer(self.audioConverter, aacEncodeInputDataProc, (__bridge void * _Nullable)(self), &dataPacketSize, &audioBufferList, NULL);
        if (status == noErr) {
            NSData *aacData = [NSData dataWithBytes:audioBufferList.mBuffers[0].mData length:audioBufferList.mBuffers[0].mDataByteSize];
            free(pcmbuffer);
            //添加ADTS头,想要获取裸流时,请忽略添加ADTS头,写入文件时,必须添加
            //                        NSData *adtsHeader = [self adtsDataForPacketLength:rawAAC.length];
            //                        NSMutableData *fullData = [NSMutableData dataWithCapacity:adtsHeader.length + rawAAC.length];;
            //                        [fullData appendData:adtsHeader];
            //                        [fullData appendData:rawAAC];
            dispatch_async(self.encodeCallbackQueue, ^{
                [self decodeAudioAACData:aacData];
            });
        }
        CFRelease(bufferRef);
        CFRelease(sampleBuffer);
    });
}

// AAC ADtS头
- (NSData*)adtsDataForPacketLength:(NSUInteger)packetLength {
    int adtsLength = 7;
    char *packet = malloc(sizeof(char) * adtsLength);
    // Variables Recycled by addADTStoPacket
    int profile = 2;  //AAC LC
    //39=MediaCodecInfo.CodecProfileLevel.AACObjectELD;
    int freqIdx = 4;  //3: 48000 Hz、4:44.1KHz、8: 16000 Hz、11: 8000 Hz
    int chanCfg = 1;  //MPEG-4 Audio Channel Configuration. 1 Channel front-center
    NSUInteger fullLength = adtsLength + packetLength;
    // fill in ADTS data
    packet[0] = (char)0xFF;    // 11111111      = syncword
    packet[1] = (char)0xF9;    // 1111 1 00 1  = syncword MPEG-2 Layer CRC
    packet[2] = (char)(((profile-1)<<6) + (freqIdx<<2) +(chanCfg>>2));
    packet[3] = (char)(((chanCfg&3)<<6) + (fullLength>>11));
    packet[4] = (char)((fullLength&0x7FF) >> 3);
    packet[5] = (char)(((fullLength&7)<<5) + 0x1F);
    packet[6] = (char)0xFC;
    NSData *data = [NSData dataWithBytesNoCopy:packet length:adtsLength freeWhenDone:YES];
    return data;
}

#pragma mark - 音频解码
// 初始化
- (void)setupEncoder {
    //输出参数pcm
    AudioStreamBasicDescription outputAudioDes = {0};
    outputAudioDes.mSampleRate = (Float64)self.sampleRate;       //采样率
    outputAudioDes.mChannelsPerFrame = (UInt32)self.channelCount; //输出声道数
    outputAudioDes.mFormatID = kAudioFormatLinearPCM;                //输出格式
    outputAudioDes.mFormatFlags = (kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked); //编码 12
    outputAudioDes.mFramesPerPacket = 1;                            //每一个packet帧数 ;
    outputAudioDes.mBitsPerChannel = 16;                             //数据帧中每个通道的采样位数。
    outputAudioDes.mBytesPerFrame = outputAudioDes.mBitsPerChannel / 8 *outputAudioDes.mChannelsPerFrame;                              //每一帧大小(采样位数 / 8 *声道数)
    outputAudioDes.mBytesPerPacket = outputAudioDes.mBytesPerFrame * outputAudioDes.mFramesPerPacket;                             //每个packet大小(帧大小 * 帧数)
    outputAudioDes.mReserved =  0;                                  //对其方式 0(8字节对齐)
    
    //输入参数aac
    AudioStreamBasicDescription inputAduioDes = {0};
    inputAduioDes.mSampleRate = (Float64)self.sampleRate;
    inputAduioDes.mFormatID = kAudioFormatMPEG4AAC;
    inputAduioDes.mFormatFlags = kMPEG4Object_AAC_LC;
    inputAduioDes.mFramesPerPacket = 1024;
    inputAduioDes.mChannelsPerFrame = (UInt32)self.channelCount;
    
    //填充输出相关信息
    UInt32 inDesSize = sizeof(inputAduioDes);
    AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, NULL, &inDesSize, &inputAduioDes);
    
    OSStatus status = AudioConverterNew(&inputAduioDes, &outputAudioDes, &_audioDecodeConverter);
    if (status != noErr) {
        NSLog(@"Error!:硬解码AAC创建失败, status= %d", (int)status);
        return;
    }
}

// 解码器回调函数
static OSStatus AudioDecoderConverterComplexInputDataProc(  AudioConverterRef inAudioConverter, UInt32 *ioNumberDataPackets, AudioBufferList *ioData,  AudioStreamPacketDescription **outDataPacketDescription,  void *inUserData) {
    
    
    CCAudioUserData *audioDecoder = (CCAudioUserData *)(inUserData);
    if (audioDecoder->size <= 0) {
        ioNumberDataPackets = 0;
        return -1;
    }
   
    //填充数据
    *outDataPacketDescription = &audioDecoder->packetDesc;
    (*outDataPacketDescription)[0].mStartOffset = 0;
    (*outDataPacketDescription)[0].mDataByteSize = audioDecoder->size;
    (*outDataPacketDescription)[0].mVariableFramesInPacket = 0;
    
    ioData->mBuffers[0].mData = audioDecoder->data;
    ioData->mBuffers[0].mDataByteSize = audioDecoder->size;
    ioData->mBuffers[0].mNumberChannels = audioDecoder->channelCount;
    
    return noErr;
}

- (void)decodeAudioAACData:(NSData *)aacData {
   
    if (!_audioDecodeConverter) {
        [self setupEncoder];
    }
    
    dispatch_async(self.decodeQueue, ^{
     
        //记录aac 作为参数参入解码回调函数
        CCAudioUserData userData = {0};
        userData.channelCount = (UInt32)self.channelCount;
        userData.data = (char *)[aacData bytes];
        userData.size = (UInt32)aacData.length;
        userData.packetDesc.mDataByteSize = (UInt32)aacData.length;
        userData.packetDesc.mStartOffset = 0;
        userData.packetDesc.mVariableFramesInPacket = 0;
        
        //输出大小和packet个数
        UInt32 pcmBufferSize = (UInt32)(2048 * self.channelCount);
        UInt32 pcmDataPacketSize = 1024;
        
        //创建临时容器pcm
        uint8_t *pcmBuffer = malloc(pcmBufferSize);
        memset(pcmBuffer, 0, pcmBufferSize);
        
        //输出buffer
        AudioBufferList outAudioBufferList = {0};
        outAudioBufferList.mNumberBuffers = 1;
        outAudioBufferList.mBuffers[0].mNumberChannels = (uint32_t)self.channelCount;
        outAudioBufferList.mBuffers[0].mDataByteSize = (UInt32)pcmBufferSize;
        outAudioBufferList.mBuffers[0].mData = pcmBuffer;
        
        //输出描述
        AudioStreamPacketDescription outputPacketDesc = {0};
        
        //配置填充函数,获取输出数据
        OSStatus status = AudioConverterFillComplexBuffer(self->_audioDecodeConverter, AudioDecoderConverterComplexInputDataProc, &userData, &pcmDataPacketSize, &outAudioBufferList, &outputPacketDesc);
        if (status != noErr) {
            NSLog(@"Error: AAC Decoder error, status=%d",(int)status);
            return;
        }
        //如果获取到数据
        if (outAudioBufferList.mBuffers[0].mDataByteSize > 0) {
            NSData *rawData = [NSData dataWithBytes:outAudioBufferList.mBuffers[0].mData length:outAudioBufferList.mBuffers[0].mDataByteSize];
            dispatch_async(self.decodeCallbackQueue, ^{
                // 这里可以处理解码出来的pcm数据
                NSLog(@"%@",rawData);
            });
        }
        free(pcmBuffer);
    });
    
}

#pragma mark - AVCaptureAudioDataOutputSampleBufferDelegate
- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection{
    if (connection == self.audioConnettion) {
        [self setAudioSampleBuffer:sampleBuffer];
    }
}

#pragma mark - 析构
- (void)dealloc {
    if (_audioConverter) {
        AudioConverterDispose(_audioConverter);
        _audioConverter = NULL;
    }
    if (_audioDecodeConverter) {
        AudioConverterDispose(_audioDecodeConverter);
        _audioDecodeConverter = NULL;
    }
}

@end