[camera_avfoundation] Fix crash when streaming while recording

packages/camera/camera_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.21
+
+* Fixes crash when streaming is enabled during recording.
+
 ## 0.9.20+7
 
 * Updates kotlin version to 2.2.0 to enable gradle 8.11 support.

packages/camera/camera_avfoundation/example/ios/RunnerTests/StreamingTests.swift

@@ -36,6 +36,7 @@ final class StreamingTests: XCTestCase {
     DefaultCamera,
     AVCaptureOutput,
     CMSampleBuffer,
+    CMSampleBuffer,
     AVCaptureConnection
   ) {
     let captureSessionQueue = DispatchQueue(label: "testing")
@@ -45,13 +46,14 @@ final class StreamingTests: XCTestCase {
     let camera = CameraTestUtils.createTestCamera(configuration)
     let testAudioOutput = CameraTestUtils.createTestAudioOutput()
     let sampleBuffer = CameraTestUtils.createTestSampleBuffer()
+    let audioSampleBuffer = CameraTestUtils.createTestAudioSampleBuffer()
     let testAudioConnection = CameraTestUtils.createTestConnection(testAudioOutput)
 
-    return (camera, testAudioOutput, sampleBuffer, testAudioConnection)
+    return (camera, testAudioOutput, sampleBuffer, audioSampleBuffer, testAudioConnection)
   }
 
   func testExceedMaxStreamingPendingFramesCount() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
     let handlerMock = MockImageStreamHandler()
 
     let finishStartStreamExpectation = expectation(
@@ -87,7 +89,7 @@ final class StreamingTests: XCTestCase {
   }
 
   func testReceivedImageStreamData() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
     let handlerMock = MockImageStreamHandler()
 
     let finishStartStreamExpectation = expectation(
@@ -124,8 +126,34 @@ final class StreamingTests: XCTestCase {
     waitForExpectations(timeout: 30, handler: nil)
   }
 
+  func testIgnoresNonImageBuffers() {
+    let (camera, testAudioOutput, _, audioSampleBuffer, testAudioConnection) = createCamera()
+    let handlerMock = MockImageStreamHandler()
+    handlerMock.eventSinkStub = { event in
+      XCTFail()
+    }
+
+    let finishStartStreamExpectation = expectation(
+      description: "Finish startStream")
+
+    let messenger = MockFlutterBinaryMessenger()
+    camera.startImageStream(
+      with: messenger, imageStreamHandler: handlerMock,
+      completion: {
+        _ in
+        finishStartStreamExpectation.fulfill()
+      })
+
+    waitForExpectations(timeout: 30, handler: nil)
+    XCTAssertEqual(camera.isStreamingImages, true)
+
+    camera.captureOutput(testAudioOutput, didOutput: audioSampleBuffer, from: testAudioConnection)
+
+    waitForQueueRoundTrip(with: DispatchQueue.main)
+  }
+
   func testImageStreamEventFormat() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
 
     let expectation = expectation(description: "Received a valid event")
 

packages/camera/camera_avfoundation/ios/camera_avfoundation/Sources/camera_avfoundation/DefaultCamera.swift

@@ -758,73 +758,7 @@ final class DefaultCamera: FLTCam, Camera {
       return
     }
 
-    if isStreamingImages {
-      if let eventSink = imageStreamHandler?.eventSink,
-        streamingPendingFramesCount < maxStreamingPendingFramesCount
-      {
-        streamingPendingFramesCount += 1
-
-        let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)!
-        // Must lock base address before accessing the pixel data
-        CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
-
-        let imageWidth = CVPixelBufferGetWidth(pixelBuffer)
-        let imageHeight = CVPixelBufferGetHeight(pixelBuffer)
-
-        var planes: [[String: Any]] = []
-
-        let isPlanar = CVPixelBufferIsPlanar(pixelBuffer)
-        let planeCount = isPlanar ? CVPixelBufferGetPlaneCount(pixelBuffer) : 1
-
-        for i in 0..<planeCount {
-          let planeAddress: UnsafeMutableRawPointer?
-          let bytesPerRow: Int
-          let height: Int
-          let width: Int
-
-          if isPlanar {
-            planeAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i)
-            bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i)
-            height = CVPixelBufferGetHeightOfPlane(pixelBuffer, i)
-            width = CVPixelBufferGetWidthOfPlane(pixelBuffer, i)
-          } else {
-            planeAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
-            bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
-            height = CVPixelBufferGetHeight(pixelBuffer)
-            width = CVPixelBufferGetWidth(pixelBuffer)
-          }
-
-          let length = bytesPerRow * height
-          let bytes = Data(bytes: planeAddress!, count: length)
-
-          let planeBuffer: [String: Any] = [
-            "bytesPerRow": bytesPerRow,
-            "width": width,
-            "height": height,
-            "bytes": FlutterStandardTypedData(bytes: bytes),
-          ]
-          planes.append(planeBuffer)
-        }
-
-        // Lock the base address before accessing pixel data, and unlock it afterwards.
-        // Done accessing the `pixelBuffer` at this point.
-        CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
-
-        let imageBuffer: [String: Any] = [
-          "width": imageWidth,
-          "height": imageHeight,
-          "format": videoFormat,
-          "planes": planes,
-          "lensAperture": Double(captureDevice.lensAperture()),
-          "sensorExposureTime": Int(captureDevice.exposureDuration().seconds * 1_000_000_000),
-          "sensorSensitivity": Double(captureDevice.iso()),
-        ]
-
-        DispatchQueue.main.async {
-          eventSink(imageBuffer)
-        }
-      }
-    }
+    handleSampleBufferStreaming(sampleBuffer)
 
     if isRecording && !isRecordingPaused {
       if videoWriter?.status == .failed, let error = videoWriter?.error {
@@ -905,6 +839,81 @@ final class DefaultCamera: FLTCam, Camera {
     }
   }
 
+  private func handleSampleBufferStreaming(_ sampleBuffer: CMSampleBuffer) {
+    guard isStreamingImages,
+      let eventSink = imageStreamHandler?.eventSink,
+      streamingPendingFramesCount < maxStreamingPendingFramesCount
+    else {
+      return
+    }
+
+    // Non-pixel buffer samples, such as audio samples, are ignored for streaming
+    guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
+      return
+    }
+
+    streamingPendingFramesCount += 1
+
+    // Must lock base address before accessing the pixel data
+    CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
+
+    let imageWidth = CVPixelBufferGetWidth(pixelBuffer)
+    let imageHeight = CVPixelBufferGetHeight(pixelBuffer)
+
+    var planes: [[String: Any]] = []
+
+    let isPlanar = CVPixelBufferIsPlanar(pixelBuffer)
+    let planeCount = isPlanar ? CVPixelBufferGetPlaneCount(pixelBuffer) : 1
+
+    for i in 0..<planeCount {
+      let planeAddress: UnsafeMutableRawPointer?
+      let bytesPerRow: Int
+      let height: Int
+      let width: Int
+
+      if isPlanar {
+        planeAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i)
+        bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i)
+        height = CVPixelBufferGetHeightOfPlane(pixelBuffer, i)
+        width = CVPixelBufferGetWidthOfPlane(pixelBuffer, i)
+      } else {
+        planeAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
+        bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
+        height = CVPixelBufferGetHeight(pixelBuffer)
+        width = CVPixelBufferGetWidth(pixelBuffer)
+      }
+
+      let length = bytesPerRow * height
+      let bytes = Data(bytes: planeAddress!, count: length)
+
+      let planeBuffer: [String: Any] = [
+        "bytesPerRow": bytesPerRow,
+        "width": width,
+        "height": height,
+        "bytes": FlutterStandardTypedData(bytes: bytes),
+      ]
+      planes.append(planeBuffer)
+    }
+
+    // Lock the base address before accessing pixel data, and unlock it afterwards.
+    // Done accessing the `pixelBuffer` at this point.
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
+
+    let imageBuffer: [String: Any] = [
+      "width": imageWidth,
+      "height": imageHeight,
+      "format": videoFormat,
+      "planes": planes,
+      "lensAperture": Double(captureDevice.lensAperture()),
+      "sensorExposureTime": Int(captureDevice.exposureDuration().seconds * 1_000_000_000),
+      "sensorSensitivity": Double(captureDevice.iso()),
+    ]
+
+    DispatchQueue.main.async {
+      eventSink(imageBuffer)
+    }
+  }
+
   private func copySampleBufferWithAdjustedTime(_ sample: CMSampleBuffer, by offset: CMTime)
     -> CMSampleBuffer?
   {

packages/camera/camera_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_avfoundation
 description: iOS implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.20+7
+version: 0.9.21
 
 environment:
   sdk: ^3.6.0