当前位置： 首页 > news >正文

iOS使用Metal对采集视频进行渲染

news 来源：原创 2025/5/24 5:34:38

在这里插入图片描述
首先对于每一帧的采集成功后，都会有一个闭包回调，然后调用renderPixelBuffer

再看renderPixelBuffer的实现在这里插入图片描述
会调用draw方法

// MARK: - MTKViewDelegate
extension KFMetalView: MTKViewDelegate {func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {viewportSize = vector_uint2(UInt32(size.width), UInt32(size.height))}func draw(in view: MTKView) {// Metal 视图回调，有数据情况下渲染视图weak var weakSelf = selfrenderQueue.async {guard let self = weakSelf else { return }self.drawInMTKView(view)}}
}

最后看向真正的实现：drawInMTKView

/// 渲染数据private func drawInMTKView(_ view: MTKView) {semaphore.wait()defer { semaphore.signal() }guard let pixelBuffer = self.pixelBuffer else { return }self.pixelBuffer = nil// 检查并初始化命令队列if commandQueue == nil {guard let device = mtkView.device else {return}commandQueue = device.makeCommandQueue()}guard let commandQueue = self.commandQueue,let renderPassDescriptor = view.currentRenderPassDescriptor,let drawable = view.currentDrawable else { return }let commandBuffer = commandQueue.makeCommandBuffer()guard let renderEncoder = commandBuffer?.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { return }renderEncoder.setViewport(MTLViewport(originX: 0.0, originY: 0.0,width: Double(viewportSize.x),height: Double(viewportSize.y),znear: -1.0, zfar: 1.0))let isRenderYUV = CVPixelBufferGetPlaneCount(pixelBuffer) > 1// 确保创建管道状态if pipelineState == nil {setupPipeline(isYUV: isRenderYUV)}guard let pipelineState = pipelineState else {renderEncoder.endEncoding()return}renderEncoder.setRenderPipelineState(pipelineState)if updateFillMode {updateVertices()updateFillMode = false}if let vertices = vertices {// 使用常量值0作为顶点缓冲索引，与Metal着色器中定义一致renderEncoder.setVertexBuffer(vertices, offset: 0, index: 0)} else {renderEncoder.endEncoding()return}if isRenderYUV {// 处理 YUV 纹理var textureY: MTLTexture?var textureUV: MTLTexture?if let textureCache = textureCache {// 确保pixelBuffer被锁定以便安全访问CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)defer {CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)}let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0)let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)let pixelFormat = MTLPixelFormat.r8Unormvar cvTextureY: CVMetalTexture?let statusY = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvTextureY)if statusY == kCVReturnSuccess, let cvTextureY = cvTextureY {textureY = CVMetalTextureGetTexture(cvTextureY)}let widthUV = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1)let heightUV = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1)let pixelFormatUV = MTLPixelFormat.rg8Unormvar cvTextureUV: CVMetalTexture?let statusUV = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormatUV, widthUV, heightUV, 1, &cvTextureUV)if statusUV == kCVReturnSuccess, let cvTextureUV = cvTextureUV {textureUV = CVMetalTextureGetTexture(cvTextureUV)}}if let textureY = textureY, let textureUV = textureUV {// 设置纹理，使用shader中定义的索引renderEncoder.setFragmentTexture(textureY, index: 0)renderEncoder.setFragmentTexture(textureUV, index: 1)} else {renderEncoder.endEncoding()return}if yuvMatrix == nil {// 获取颜色空间信息，如果没有就使用默认值let colorSpace: CFTypeReflet isFullRange: Bool// 尝试从pixelBuffer获取YUV颜色空间信息if let matrixKey = CVBufferCopyAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, nil) {colorSpace = matrixKeyisFullRange = pixelBufferIsFullRange(pixelBuffer)} else {// 如果pixelBuffer中没有颜色空间信息，使用默认值// 对于相机视频，通常使用BT.601标准和full rangecolorSpace = kCVImageBufferYCbCrMatrix_ITU_R_601_4// 检查格式类型判断是否为full rangelet format = CVPixelBufferGetPixelFormatType(pixelBuffer)isFullRange = (format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)}setupYUVMatrix(isFullRange: isFullRange, colorSpace: colorSpace)}// 设置矩阵缓冲区，使用shader中定义的索引if let yuvMatrix = yuvMatrix {renderEncoder.setFragmentBuffer(yuvMatrix, offset: 0, index: 0)}} else {// 处理 RGB 纹理var textureRGB: MTLTexture?if let textureCache = textureCache {let width = CVPixelBufferGetWidth(pixelBuffer)let height = CVPixelBufferGetHeight(pixelBuffer)let pixelFormat = MTLPixelFormat.bgra8Unormvar cvTextureRGB: CVMetalTexture?let status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvTextureRGB)if status == kCVReturnSuccess, let cvTextureRGB = cvTextureRGB {textureRGB = CVMetalTextureGetTexture(cvTextureRGB)}}if let textureRGB = textureRGB {// 设置RGB纹理，使用shader中定义的索引renderEncoder.setFragmentTexture(textureRGB, index: 0)} else {renderEncoder.endEncoding()return}}renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: numVertices)renderEncoder.endEncoding()commandBuffer?.present(drawable)commandBuffer?.commit()}

drawInMTKView 函数的实现过程总结：

  1. 获取并清除像素缓冲区

    • 使用信号量确保线程安全访问
    • 获取当前的 pixelBuffer 并立即清除引用

  2. 准备渲染资源

    • 检查并初始化命令队列
    • 获取当前渲染描述符和可绘制对象
    • 创建命令缓冲区和渲染编码器

  3. 设置渲染状态

    • 设置视口大小
    • 检测是否为 YUV 格式（通过平面数量判断）
    • 确保渲染管道状态已创建（如未创建则调用 setupPipeline）
    • 检查顶点数据是否需要更新（根据填充模式）

  4. 设置顶点缓冲区

    • 将顶点数据传递给渲染编码器

  5. 处理纹理

    • 对于 YUV 格式：
      • 创建 Y 平面和 UV 平面的纹理
      • 设置 YUV 转换矩阵（如果尚未设置）
      • 将纹理和矩阵传递给片段着色器
    • 对于 RGB 格式：
      • 创建单一 RGB 纹理并传递给片段着色器

  6. 执行渲染

    • 绘制三角形带（triangleStrip）
    • 结束编码
    • 提交命令缓冲区并呈现到屏幕

  7. 错误处理

    • 在各个步骤中检查资源是否正确创建，如果失败则提前结束渲染过程

整个过程实现了从 CVPixelBuffer（可能是 YUV 或 RGB 格式）到 Metal 渲染的完整流程，包括纹理创建、格式转换和根据填充模式调整显示比例。

对于渲染的其他部分，是基于渲染管线进行渲染的，其他操作和渲染三角形之类的大差不差，比如配置Metal文件，创建渲染管线等。

//
//  Shader.metal
//  VideoDemo
//
//  Created by ricard.li on 2025/5/23.
//#include <metal_stdlib>
using namespace metal;typedef struct {// 顶点坐标，4 维向量。vector_float4 position;// 纹理坐标。vector_float2 textureCoordinate;
} KFVertex;typedef struct {// YUV 矩阵，使用float4x4替代float3x3以确保内存对齐float4x4 matrix;// 是否为 full range，使用uint代替bool以确保内存布局一致uint fullRange;
} KFConvertMatrix;// 顶点的桥接枚举值 KFVertexInputIndexVertices。
typedef enum KFVertexInputIndex {KFVertexInputIndexVertices = 0,
} KFVertexInputIndex;// YUV 矩阵的桥接枚举值 KFFragmentInputIndexMatrix。
typedef enum KFFragmentBufferIndex {KFFragmentInputIndexMatrix = 0,
} KFMetalFragmentBufferIndex;// YUV 数据的桥接枚举值 KFFragmentTextureIndexTextureY、KFFragmentTextureIndexTextureUV。
typedef enum KFFragmentYUVTextureIndex {KFFragmentTextureIndexTextureY = 0,KFFragmentTextureIndexTextureUV = 1,
} KFFragmentYUVTextureIndex;// RGBA 数据的桥接枚举值 KFFragmentTextureIndexTextureRGB。
typedef enum KFFragmentRGBTextureIndex {KFFragmentTextureIndexTextureRGB = 0,
} KFFragmentRGBTextureIndex;// 定义了一个类型为 RasterizerData 的结构体，里面有一个 float4 向量和 float2 向量。
typedef struct {// float4：4 维向量；// clipSpacePosition：参数名，表示顶点；// [[position]]：position 是顶点修饰符，这是苹果内置的语法，不能改变，表示顶点信息。float4 clipSpacePosition [[position]];// float2：2 维向量；// textureCoordinate：参数名，这里表示纹理。float2 textureCoordinate;
} RasterizerData;// 顶点函数通过一个自定义的结构体，返回对应的数据；顶点函数的输入参数也可以是自定义结构体。// 顶点函数
// vertex：函数修饰符，表示顶点函数；
// RasterizerData：返回值类型；
// vertexShader：函数名；
// [[vertex_id]]：vertex_id 是顶点 id 修饰符，苹果内置的语法不可改变；
// [[buffer(YYImageVertexInputIndexVertexs)]]：buffer 是缓存数据修饰符，苹果内置的语法不可改变，YYImageVertexInputIndexVertexs 是索引；
// constant：是变量类型修饰符，表示存储在 device 区域。
vertex RasterizerData vertexShader(uint vertexID [[vertex_id]],constant KFVertex *vertexArray [[buffer(KFVertexInputIndexVertices)]]) {RasterizerData out;out.clipSpacePosition = vertexArray[vertexID].position;out.textureCoordinate = vertexArray[vertexID].textureCoordinate;return out;
}// 片元函数
// fragment：函数修饰符，表示片元函数；
// float4：返回值类型，返回 RGBA；
// fragmentImageShader：函数名；
// RasterizerData：参数类型；
// input：变量名；
// [[stage_in]：stage_in 表示这个数据来自光栅化，光栅化是顶点处理之后的步骤，业务层无法修改。
// texture2d：类型表示纹理；
// textureY：表示 Y 通道；
// textureUV：表示 UV 通道；
// [[texture(index)]]：纹理修饰符；可以加索引：[[texture(0)]] 对应纹理 0，[[texture(1)]] 对应纹理 1；
// KFFragmentTextureIndexTextureY、KFFragmentTextureIndexTextureUV：表示纹理索引。
fragment float4 yuvSamplingShader(RasterizerData input [[stage_in]],texture2d<float> textureY [[texture(KFFragmentTextureIndexTextureY)]],texture2d<float> textureUV [[texture(KFFragmentTextureIndexTextureUV)]],constant KFConvertMatrix *convertMatrix [[buffer(KFFragmentInputIndexMatrix)]]) {constexpr sampler textureSampler (mag_filter::linear, min_filter::linear);// 初始化YUV向量float3 yuv;// 根据范围处理Y值if (convertMatrix->fullRange != 0) { // full range.yuv.x = textureY.sample(textureSampler, input.textureCoordinate).r;} else { // video range.yuv.x = textureY.sample(textureSampler, input.textureCoordinate).r - (16.0 / 255.0);}// 获取UV值，并从[0,1]范围转换到[-0.5,0.5]yuv.yz = textureUV.sample(textureSampler, input.textureCoordinate).rg - 0.5;// 使用YUV转RGB矩阵计算RGB值 - 使用float4x4矩阵，但仍提取float3结果float3 rgb = float3(convertMatrix->matrix * float4(yuv, 1.0));// 确保结果在合法范围内rgb = clamp(rgb, 0.0, 1.0);return float4(rgb, 1.0);
}// 片元函数
// fragment：函数修饰符，表示片元函数；
// float4：返回值类型，返回 RGBA；
// fragmentImageShader：函数名；
// RasterizerData：参数类型；
// input：变量名；
// [[stage_in]]：stage_in 表示这个数据来自光栅化，光栅化是顶点处理之后的步骤，业务层无法修改。
// texture2d：类型表示纹理；
// colorTexture：代表 RGBA 数据；
// [[texture(index)]]：纹理修饰符；可以加索引：[[texture(0)]] 对应纹理 0，[[texture(1)]] 对应纹理 1；
// KFFragmentTextureIndexTextureRGB：表示纹理索引。
fragment float4 rgbSamplingShader(RasterizerData input [[stage_in]],texture2d<half> colorTexture [[texture(KFFragmentTextureIndexTextureRGB)]]) {constexpr sampler textureSampler (mag_filter::linear, min_filter::linear);half4 colorSample = colorTexture.sample(textureSampler, input.textureCoordinate);return float4(colorSample);
}
//
//  KFMetalView.swift
//  VideoDemo
//
//  Created by ricard.li on 2025/5/23.
//import UIKit
import MetalKit
import AVFoundation
import MetalPerformanceShaders
import simd// 渲染画面填充模式
enum KFMetalViewContentMode: Int {case stretch = 0  // 自动填充满，可能会变形case fit = 1      // 按比例适配，可能会有黑边case fill = 2     // 根据比例裁剪后填充满
}// 颜色空间转换矩阵，BT.601 Video Range
private let kFColorMatrix601VideoRange = matrix_float4x4(simd_float4(1.164, 1.164, 1.164, 0.0),simd_float4(0.0, -0.392, 2.017, 0.0),simd_float4(1.596, -0.813, 0.0, 0.0),simd_float4(0.0, 0.0, 0.0, 1.0)
)// 颜色空间转换矩阵，BT.601 Full Range
private let kFColorMatrix601FullRange = matrix_float4x4(simd_float4(1.0, 1.0, 1.0, 0.0),simd_float4(0.0, -0.343, 1.765, 0.0),simd_float4(1.4, -0.711, 0.0, 0.0),simd_float4(0.0, 0.0, 0.0, 1.0)
)// 颜色空间转换矩阵，BT.709 Video Range
private let kFColorMatrix709VideoRange = matrix_float4x4(simd_float4(1.164, 1.164, 1.164, 0.0),simd_float4(0.0, -0.213, 2.112, 0.0),simd_float4(1.793, -0.533, 0.0, 0.0),simd_float4(0.0, 0.0, 0.0, 1.0)
)// 颜色空间转换矩阵，BT.709 Full Range
private let kFColorMatrix709FullRange = matrix_float4x4(simd_float4(1.0, 1.0, 1.0, 0.0),simd_float4(0.0, -0.187, 1.856, 0.0),simd_float4(1.575, -0.468, 0.0, 0.0),simd_float4(0.0, 0.0, 0.0, 1.0)
)class KFMetalView: UIView {// MARK: - Public Properties// 画面填充模式var fillMode: KFMetalViewContentMode = .fit {didSet {updateFillMode = true}}// MARK: - Private Properties// 外层输入的最后一帧数据private var pixelBuffer: CVPixelBuffer?// 处理 PixelBuffer 锁，防止外层输入线程与渲染线程同时操作 Crashprivate let semaphore = DispatchSemaphore(value: 1)// 纹理缓存，根据 pixelbuffer 获取纹理private var textureCache: CVMetalTextureCache?// Metal 渲染的 viewprivate var mtkView: MTKView!// 视口大小private var viewportSize = vector_uint2(0, 0)// 渲染管道，管理顶点函数和片元函数private var pipelineState: MTLRenderPipelineState?// 渲染指令队列private var commandQueue: MTLCommandQueue?// 顶点缓存对象private var vertices: MTLBuffer?// 顶点数量private var numVertices: Int = 0// YUV 数据矩阵对象private var yuvMatrix: MTLBuffer?// 填充模式变更标记private var updateFillMode = true// pixelBuffer 数据尺寸private var pixelBufferSize = CGSize.zero// 当前视图大小private var currentViewSize = CGSize.zero// 渲染线程private let renderQueue = DispatchQueue(label: "com.KeyFrameKit.metalView.renderQueue", qos: .userInteractive)// MARK: - Lifecycleoverride init(frame: CGRect) {super.init(frame: frame)setupView(frame: frame)}required init?(coder: NSCoder) {super.init(coder: coder)setupView(frame: bounds)}private func setupView(frame: CGRect) {currentViewSize = frame.sizefillMode = .fitupdateFillMode = true// 设置视图背景色为黑色backgroundColor = .black// 创建 Metal 渲染视图且添加到当前视图mtkView = MTKView(frame: bounds)guard let device = MTLCreateSystemDefaultDevice() else {return }mtkView.device = devicemtkView.backgroundColor = .black  // 设置Metal视图背景色为黑色// 确保使用正确的像素格式mtkView.colorPixelFormat = .bgra8Unorm// 确保每次绘制都会调用delegatemtkView.isPaused = false// 确保绘制能够连续进行mtkView.enableSetNeedsDisplay = falseaddSubview(mtkView)mtkView.delegate = selfmtkView.framebufferOnly = trueviewportSize = vector_uint2(UInt32(mtkView.drawableSize.width), UInt32(mtkView.drawableSize.height))// 创建纹理缓存var newTextureCache: CVMetalTextureCache?let status = CVMetalTextureCacheCreate(nil, nil, device, nil, &newTextureCache)if status == kCVReturnSuccess {textureCache = newTextureCache}// 预先创建命令队列commandQueue = device.makeCommandQueue()}//确保Metal视图(mtkView)与包含它的父视图(KFMetalView)大小完全一致   override func layoutSubviews() {// 视图自动调整布局，同步至 Metal 视图super.layoutSubviews()mtkView.frame = boundscurrentViewSize = bounds.size}deinit {// 释放最后一帧数据、纹理缓存semaphore.wait()pixelBuffer = nilif let textureCache = textureCache {CVMetalTextureCacheFlush(textureCache, 0)}textureCache = nilsemaphore.signal()mtkView.releaseDrawables()}// MARK: - Public Methods/// 渲染 CVPixelBufferfunc renderPixelBuffer(_ pixelBuffer: CVPixelBuffer) {semaphore.wait()self.pixelBuffer = pixelBufferpixelBufferSize = CGSize(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))semaphore.signal()// 确保在主线程调用setNeedsDisplay触发绘制DispatchQueue.main.async { [weak self] inself?.mtkView.draw()}}// MARK: - Private Methods/// 初始化渲染管道private func setupPipeline(isYUV: Bool) {guard let device = mtkView.device else { return }// 获取默认库guard let library = device.makeDefaultLibrary() else {return}// 根据是否处理YUV选择顶点和片元着色器let vertexFunctionName = "vertexShader"let fragmentFunctionName = isYUV ? "yuvSamplingShader" : "rgbSamplingShader"guard let vertexFunction = library.makeFunction(name: vertexFunctionName) else {return}guard let fragmentFunction = library.makeFunction(name: fragmentFunctionName) else {return}let pipelineStateDescriptor = MTLRenderPipelineDescriptor()pipelineStateDescriptor.vertexFunction = vertexFunctionpipelineStateDescriptor.fragmentFunction = fragmentFunctionpipelineStateDescriptor.colorAttachments[0].pixelFormat = mtkView.colorPixelFormatdo {pipelineState = try device.makeRenderPipelineState(descriptor: pipelineStateDescriptor)if commandQueue == nil {commandQueue = device.makeCommandQueue()}} catch {// 使用do-catch而不是print，避免不必要的日志输出}}/// 初始化 YUV 矩阵private func setupYUVMatrix(isFullRange: Bool, colorSpace: CFTypeRef) {guard let device = mtkView.device else { return }// 选择正确的YUV转RGB矩阵var matrix: matrix_float4x4// 安全地检查颜色空间类型let colorSpaceString = CFCopyDescription(colorSpace) as Stringlet is601 = colorSpaceString.contains("601")let is709 = colorSpaceString.contains("709")if is601 {if isFullRange {matrix = kFColorMatrix601FullRange} else {matrix = kFColorMatrix601VideoRange}} else if is709 {if isFullRange {matrix = kFColorMatrix709FullRange} else {matrix = kFColorMatrix709VideoRange}} else {// 默认使用BT.601if isFullRange {matrix = kFColorMatrix601FullRange} else {matrix = kFColorMatrix601VideoRange}}// 创建转换矩阵结构体，将Bool转换为UInt32var convertMatrix = KFConvertMatrix(matrix: matrix, fullRange: isFullRange ? 1 : 0)// 创建矩阵缓冲区yuvMatrix = device.makeBuffer(bytes: &convertMatrix, length: MemoryLayout<KFConvertMatrix>.size, options: .storageModeShared)}/// 更新顶点数据private func updateVertices() {guard let device = mtkView.device else { return }var widthScaling: Float = 1.0var heightScaling: Float = 1.0if !currentViewSize.equalTo(.zero) && !pixelBufferSize.equalTo(.zero) {// 计算视图和纹理的宽高比let viewAspect = Float(currentViewSize.width / currentViewSize.height)let textureAspect = Float(pixelBufferSize.width / pixelBufferSize.height)switch fillMode {case .stretch:// 拉伸模式，直接使用 1.0widthScaling = 1.0heightScaling = 1.0case .fit:// 适配模式，保持比例，可能有黑边if textureAspect > viewAspect {// 视频比例比视图宽，高度需要缩小widthScaling = 1.0heightScaling = viewAspect / textureAspect} else {// 视频比例比视图窄，宽度需要缩小widthScaling = textureAspect / viewAspectheightScaling = 1.0}case .fill:// 填充模式，确保没有黑边，可能会裁剪部分内容if textureAspect > viewAspect {// 视频比例比视图宽，宽度需要裁剪widthScaling = viewAspect / textureAspectheightScaling = 1.0} else {// 视频比例比视图窄，高度需要裁剪widthScaling = 1.0heightScaling = textureAspect / viewAspect}// 填充模式要确保覆盖整个屏幕，所以这里要取倒数widthScaling = 1.0 / widthScalingheightScaling = 1.0 / heightScaling}}// 创建顶点数组，与shader中的KFVertex结构一致let quadVertices: [KFVertex] = [KFVertex(position: vector_float4(-widthScaling, -heightScaling, 0.0, 1.0), textureCoordinate: vector_float2(0.0, 1.0)),KFVertex(position: vector_float4(widthScaling, -heightScaling, 0.0, 1.0), textureCoordinate: vector_float2(1.0, 1.0)),KFVertex(position: vector_float4(-widthScaling, heightScaling, 0.0, 1.0), textureCoordinate: vector_float2(0.0, 0.0)),KFVertex(position: vector_float4(widthScaling, heightScaling, 0.0, 1.0), textureCoordinate: vector_float2(1.0, 0.0))]vertices = device.makeBuffer(bytes: quadVertices, length: MemoryLayout<KFVertex>.stride * quadVertices.count, options: .storageModeShared)numVertices = quadVertices.count}/// 判断 YUV 数据是否为 full rangeprivate func pixelBufferIsFullRange(_ pixelBuffer: CVPixelBuffer) -> Bool {// 根据像素格式类型判断let pixelFormatType = CVPixelBufferGetPixelFormatType(pixelBuffer)// 检查是否是已知的全范围格式let isFullRange: Boolswitch pixelFormatType {case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:isFullRange = truecase kCVPixelFormatType_420YpCbCr8PlanarFullRange:isFullRange = truedefault:isFullRange = false}return isFullRange}/// 渲染数据private func drawInMTKView(_ view: MTKView) {semaphore.wait()defer { semaphore.signal() }guard let pixelBuffer = self.pixelBuffer else { return }self.pixelBuffer = nil// 检查并初始化命令队列if commandQueue == nil {guard let device = mtkView.device else {return}commandQueue = device.makeCommandQueue()}guard let commandQueue = self.commandQueue,let renderPassDescriptor = view.currentRenderPassDescriptor,let drawable = view.currentDrawable else { return }let commandBuffer = commandQueue.makeCommandBuffer()guard let renderEncoder = commandBuffer?.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { return }renderEncoder.setViewport(MTLViewport(originX: 0.0, originY: 0.0,width: Double(viewportSize.x),height: Double(viewportSize.y),znear: -1.0, zfar: 1.0))let isRenderYUV = CVPixelBufferGetPlaneCount(pixelBuffer) > 1// 确保创建管道状态if pipelineState == nil {setupPipeline(isYUV: isRenderYUV)}guard let pipelineState = pipelineState else {renderEncoder.endEncoding()return}renderEncoder.setRenderPipelineState(pipelineState)if updateFillMode {updateVertices()updateFillMode = false}if let vertices = vertices {// 使用常量值0作为顶点缓冲索引，与Metal着色器中定义一致renderEncoder.setVertexBuffer(vertices, offset: 0, index: 0)} else {renderEncoder.endEncoding()return}if isRenderYUV {// 处理 YUV 纹理var textureY: MTLTexture?var textureUV: MTLTexture?if let textureCache = textureCache {// 确保pixelBuffer被锁定以便安全访问CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)defer {CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)}let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0)let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)let pixelFormat = MTLPixelFormat.r8Unormvar cvTextureY: CVMetalTexture?let statusY = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvTextureY)if statusY == kCVReturnSuccess, let cvTextureY = cvTextureY {textureY = CVMetalTextureGetTexture(cvTextureY)}let widthUV = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1)let heightUV = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1)let pixelFormatUV = MTLPixelFormat.rg8Unormvar cvTextureUV: CVMetalTexture?let statusUV = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormatUV, widthUV, heightUV, 1, &cvTextureUV)if statusUV == kCVReturnSuccess, let cvTextureUV = cvTextureUV {textureUV = CVMetalTextureGetTexture(cvTextureUV)}}if let textureY = textureY, let textureUV = textureUV {// 设置纹理，使用shader中定义的索引renderEncoder.setFragmentTexture(textureY, index: 0)renderEncoder.setFragmentTexture(textureUV, index: 1)} else {renderEncoder.endEncoding()return}if yuvMatrix == nil {// 获取颜色空间信息，如果没有就使用默认值let colorSpace: CFTypeReflet isFullRange: Bool// 尝试从pixelBuffer获取YUV颜色空间信息if let matrixKey = CVBufferCopyAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, nil) {colorSpace = matrixKeyisFullRange = pixelBufferIsFullRange(pixelBuffer)} else {// 如果pixelBuffer中没有颜色空间信息，使用默认值// 对于相机视频，通常使用BT.601标准和full rangecolorSpace = kCVImageBufferYCbCrMatrix_ITU_R_601_4// 检查格式类型判断是否为full rangelet format = CVPixelBufferGetPixelFormatType(pixelBuffer)isFullRange = (format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)}setupYUVMatrix(isFullRange: isFullRange, colorSpace: colorSpace)}// 设置矩阵缓冲区，使用shader中定义的索引if let yuvMatrix = yuvMatrix {renderEncoder.setFragmentBuffer(yuvMatrix, offset: 0, index: 0)}} else {// 处理 RGB 纹理var textureRGB: MTLTexture?if let textureCache = textureCache {let width = CVPixelBufferGetWidth(pixelBuffer)let height = CVPixelBufferGetHeight(pixelBuffer)let pixelFormat = MTLPixelFormat.bgra8Unormvar cvTextureRGB: CVMetalTexture?let status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvTextureRGB)if status == kCVReturnSuccess, let cvTextureRGB = cvTextureRGB {textureRGB = CVMetalTextureGetTexture(cvTextureRGB)}}if let textureRGB = textureRGB {// 设置RGB纹理，使用shader中定义的索引renderEncoder.setFragmentTexture(textureRGB, index: 0)} else {renderEncoder.endEncoding()return}}renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: numVertices)renderEncoder.endEncoding()commandBuffer?.present(drawable)commandBuffer?.commit()}
}// MARK: - MTKViewDelegate
extension KFMetalView: MTKViewDelegate {func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {viewportSize = vector_uint2(UInt32(size.width), UInt32(size.height))}func draw(in view: MTKView) {// Metal 视图回调，有数据情况下渲染视图weak var weakSelf = selfrenderQueue.async {guard let self = weakSelf else { return }self.drawInMTKView(view)}}
}

相关文章：

  • OpenHarmony外设驱动使用 （十三），Vibrator
  • Java桌面应用开发详解：自制截图工具从设计到打包的全流程【附源码与演示】
  • 2025年渗透测试面试题总结-匿名[社招]安全工程师(红队方向)2（题目+回答）
  • Linux（5）——再谈操作系统
  • 【AS32X601驱动系列教程】SMU_系统时钟详解
  • RNN GRU LSTM 模型理解
  • 飞桨（PaddlePaddle）在机器学习全流程（数据采集、处理、标注、建模、分析、优化）
  • 前端vue2-完全前端生成pdf-＞pdf-lib,html2canvas+jspdf,原生打印，三种方式（打印带有echarts图的pdf）
  • 可视化大屏实现全屏或非全屏
  • 继电保护与安全自动装置：电力系统安全的守护神
  • Windows 安装 FFmpeg 新手教程（附环境变量配置）
  • ProfiNet转Ethernet/IP网关选型策略适配西门子S7-1500与三菱变频器的关键参数对比
  • Oracle Apps R12——报表入门2：单表——报表开发流程
  • .NET外挂系列：6. harmony中一些实用的反射工具包
  • 大模型高效微调方法综述：P-Tuning软提示与lora低秩微调附案例代码详解
  • word设置如“第xx页 共xx页”格式的页码
  • 本地分支git push 报错 fatal: The current branch XXXX has no upstream branch.
  • 百千鳥VF可变字体 momochidori variable font
  • Lyra学习笔记1地图角色加载流程
  • 塔能高温冰蓄冷技术：工厂能耗精准节能的创新之路
  • 做网站开发学什么语言/长春网络优化哪个公司在做
  • wordpress 大站点/苏州搜索引擎排名优化商家
  • 做b2b网站有没有用/在百度上怎么注册网站
  • 查建设公司人员是那个网站/企业培训有哪些方面
  • 工商核名在哪个网站/合肥今天的最新消息
  • wordpress更多的模板/seo的优化方向