[Enhancement]Implement static frame pipeline for PiP

ipavlidakis · ipavlidakis · commit 3a3058fdff19 · 2025-03-27T18:10:04.000+02:00
diff --git a/Sources/StreamVideo/Utils/SerialActor/SerialActorQueue.swift b/Sources/StreamVideo/Utils/SerialActor/SerialActorQueue.swift
@@ -18,6 +18,12 @@ public final class SerialActorQueue: Sendable {
     /// Initializes a new `SerialActorQueue` instance.
     public init() {}
 
+    private let disposableBag = DisposableBag()
+
+    public func cancelAll() {
+        disposableBag.removeAll()
+    }
+
     /// Submits an asynchronous task to be executed serially.
     ///
     /// - Parameters:
@@ -46,7 +52,7 @@ public final class SerialActorQueue: Sendable {
                     lineNumber: line
                 )
             }
-        }
+        }.store(in: disposableBag)
     }
 
     /// Submits an asynchronous task to be executed serially and waits for it to complete.
diff --git a/Sources/StreamVideoSwiftUI/CallView/CallParticipantImageView.swift b/Sources/StreamVideoSwiftUI/CallView/CallParticipantImageView.swift
@@ -48,3 +48,27 @@ public struct CallParticipantImageView<Factory: ViewFactory>: View {
         )
     }
 }
+
+public struct PictureInPictureParticipantImageView: View {
+
+    @Injected(\.colors) var colors
+
+    var imageURL: URL?
+    var content: AnyView
+
+    public init(
+        imageURL: URL? = nil,
+        @ViewBuilder contentProvider: () -> some View
+    ) {
+        self.imageURL = imageURL
+        content = AnyView(contentProvider())
+    }
+
+    public var body: some View {
+        StreamLazyImage(imageURL: imageURL) {
+            Color(colors.participantBackground)
+        }
+        .blur(radius: 8)
+        .overlay(content)
+    }
+}
diff --git a/Sources/StreamVideoSwiftUI/CallingViews/CallingGroupView.swift b/Sources/StreamVideoSwiftUI/CallingViews/CallingGroupView.swift
@@ -148,16 +148,30 @@ struct CircledTitleView: View {
     var size: CGFloat = .expandedAvatarSize
     
     var body: some View {
-        ZStack {
-            Circle()
-                .foregroundColor(colors.tintColor)
-            Text(title)
-                .foregroundColor(.white)
-                .font(fonts.title)
-                .minimumScaleFactor(0.4)
-                .padding()
+        if size > 0 {
+            ZStack {
+                Circle()
+                    .foregroundColor(colors.tintColor)
+                Text(title)
+                    .foregroundColor(.white)
+                    .font(fonts.title)
+                    .minimumScaleFactor(0.4)
+                    .padding()
+            }
+            .frame(maxWidth: size, maxHeight: size)
+            .modifier(ShadowModifier())
+        } else {
+            ZStack {
+                Circle()
+                    .foregroundColor(colors.tintColor)
+                Text(title)
+                    .foregroundColor(.white)
+                    .font(fonts.title)
+                    .minimumScaleFactor(0.4)
+                    .padding()
+            }
+            .padding()
+            .modifier(ShadowModifier())
         }
-        .frame(maxWidth: size, maxHeight: size)
-        .modifier(ShadowModifier())
     }
 }
diff --git a/Sources/StreamVideoSwiftUI/Utils/Extensions/CVPixelBuffer+UIImage.swift b/Sources/StreamVideoSwiftUI/Utils/Extensions/CVPixelBuffer+UIImage.swift
@@ -0,0 +1,56 @@
+//
+// Copyright © 2025 Stream.io Inc. All rights reserved.
+//
+
+#if canImport(UIKit)
+import Foundation
+import UIKit
+
+extension CVPixelBuffer {
+    static func build(from uiImage: UIImage) -> CVPixelBuffer? {
+        let width = Int(uiImage.size.width)
+        let height = Int(uiImage.size.height)
+
+        var pixelBuffer: CVPixelBuffer?
+        let status = CVPixelBufferCreate(
+            kCFAllocatorDefault,
+            width,
+            height,
+            kCVPixelFormatType_32BGRA,
+            [
+                kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue,
+                kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue
+            ] as CFDictionary,
+            &pixelBuffer
+        )
+
+        guard
+            let cgImage = uiImage.cgImage,
+            status == kCVReturnSuccess,
+            let pixelBuffer = pixelBuffer
+        else {
+            return nil
+        }
+
+        CVPixelBufferLockBaseAddress(pixelBuffer, [])
+        defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) }
+
+        let context = CGContext(
+            data: CVPixelBufferGetBaseAddress(pixelBuffer),
+            width: width,
+            height: height,
+            bitsPerComponent: 8,
+            bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer),
+            space: CGColorSpaceCreateDeviceRGB(),
+            bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue
+        )
+
+        context?.draw(
+            cgImage,
+            in: CGRect(x: 0, y: 0, width: width, height: height)
+        )
+
+        return pixelBuffer
+    }
+}
+#endif
diff --git a/Sources/StreamVideoSwiftUI/Utils/Extensions/View+CVPixelBuffer.swift b/Sources/StreamVideoSwiftUI/Utils/Extensions/View+CVPixelBuffer.swift
@@ -0,0 +1,28 @@
+//
+//  View+CVPixelBuffer.swift
+//  StreamVideo
+//
+//  Created by Ilias Pavlidakis on 26/3/25.
+//
+
+import Foundation
+import SwiftUI
+
+extension View {
+
+    @MainActor
+    func toPixelBuffer(contentSize: CGSize) -> CVPixelBuffer? {
+        guard #available(iOS 16.0, *) else {
+            return nil
+        }
+        let renderer = ImageRenderer(content: self)
+        renderer.proposedSize = .init(contentSize)
+        if let image = renderer.uiImage {
+            return .build(from: image)
+        } else {
+            return nil
+        }
+    }
+}
+
+extension CVBuffer: @unchecked @retroactive Sendable {}
diff --git a/Sources/StreamVideoSwiftUI/Utils/PictureInPicture/StreamPictureInPictureAdapter.swift b/Sources/StreamVideoSwiftUI/Utils/PictureInPicture/StreamPictureInPictureAdapter.swift
@@ -12,6 +12,8 @@ import UIKit
 /// Picture display.
 public final class StreamPictureInPictureAdapter: @unchecked Sendable {
 
+    @Injected(\.staticVideoSource) private var staticVideoSource
+
     /// The active call.
     public var call: Call? {
         didSet {
@@ -36,6 +38,7 @@ public final class StreamPictureInPictureAdapter: @unchecked Sendable {
         didSet {
             Task { @MainActor in
                 pictureInPictureController?.onSizeUpdate = { [weak self] size in
+                    self?.staticVideoSource.contentSize = size
                     if let activeParticipant = self?.activeParticipant {
                         self?.onSizeUpdate?(size, activeParticipant)
                     }
@@ -89,16 +92,62 @@ public final class StreamPictureInPictureAdapter: @unchecked Sendable {
             let sessionId = call?.state.sessionId
             let otherParticipants = participants.filter { $0.sessionId != sessionId }
 
-            if let session = call?.state.screenSharingSession, call?.state.isCurrentUserScreensharing == false,
-               let track = session.track {
+            if
+                let session = call?.state.screenSharingSession, call?.state.isCurrentUserScreensharing == false,
+                let track = session.track {
                 pictureInPictureController?.track = track
+                staticVideoSource.participant = nil
                 activeParticipant = nil
-            } else if let participant = otherParticipants.first(where: { $0.track != nil }), let track = participant.track {
+
+                log.debug(
+                    "Active participant:\(session.participant.name) with screensharing will be used.",
+                    subsystems: .pictureInPicture
+                )
+            } else if
+                let participant = otherParticipants.first(where: { $0.hasVideo && $0.track != nil }),
+                let track = participant.track {
                 pictureInPictureController?.track = track
+                staticVideoSource.participant = nil
                 activeParticipant = participant
-            } else if let localParticipant = call?.state.localParticipant, let track = localParticipant.track {
+
+                log.debug(
+                    "Active participant:\(participant.name) will be used.",
+                    subsystems: .pictureInPicture
+                )
+
+            } else if
+                let participant = call?.state.dominantSpeaker {
+                pictureInPictureController?.track = nil
+                staticVideoSource.participant = participant
+                activeParticipant = participant
+
+                log.debug(
+                    "Dominant speaker participant:\(participant.name) will be used.",
+                    subsystems: .pictureInPicture
+                )
+
+            } else if
+                let localParticipant = call?.state.localParticipant,
+                localParticipant.hasVideo,
+                let track = localParticipant.track {
                 pictureInPictureController?.track = track
+                staticVideoSource.participant = nil
                 activeParticipant = localParticipant
+
+                log.debug(
+                    "Local participant:\(localParticipant.name) will be used.",
+                    subsystems: .pictureInPicture
+                )
+
+            } else {
+                pictureInPictureController?.track = nil
+                staticVideoSource.participant = otherParticipants.first
+                activeParticipant = otherParticipants.first
+
+                log.debug(
+                    "No active participant found. Will use the first participant:\(otherParticipants.first?.name ?? "n/a") in the list.",
+                    subsystems: .pictureInPicture
+                )
             }
         }
     }
diff --git a/Sources/StreamVideoSwiftUI/Utils/PictureInPicture/StreamPictureInPictureVideoRenderer.swift b/Sources/StreamVideoSwiftUI/Utils/PictureInPicture/StreamPictureInPictureVideoRenderer.swift
@@ -10,6 +10,8 @@ import StreamWebRTC
 /// A view that can be used to render an instance of `RTCVideoTrack`
 final class StreamPictureInPictureVideoRenderer: UIView, RTCVideoRenderer {
 
+    @Injected(\.staticVideoSource) private var staticVideoSource
+
     /// The rendering track.
     var track: RTCVideoTrack? {
         didSet {
@@ -47,6 +49,7 @@ final class StreamPictureInPictureVideoRenderer: UIView, RTCVideoRenderer {
 
     /// The cancellable used to control the bufferPublisher stream.
     private var bufferUpdatesCancellable: AnyCancellable?
+    private var staticFrameGenerationCancellable: AnyCancellable?
 
     /// The view's size.
     /// - Note: We are using this property instead for `frame.size` or `bounds.size` so we can
@@ -185,18 +188,16 @@ final class StreamPictureInPictureVideoRenderer: UIView, RTCVideoRenderer {
     private func process(_ buffer: CMSampleBuffer) {
         guard
             bufferUpdatesCancellable != nil,
-            let trackId = track?.trackId,
             buffer.isValid
         else {
             contentView.renderingComponent.flush()
             logMessage(.debug, message: "🔥 Display layer flushed.")
             return
         }
 
-        logMessage(
-            .debug,
-            message: "⚙️ Processing buffer for trackId:\(trackId)."
-        )
+        let trackId = track?.trackId ?? "-"
+
+        logMessage(.debug, message: "⚙️ Processing buffer for trackId:\(trackId).")
         if #available(iOS 14.0, *) {
             if contentView.renderingComponent.requiresFlushToResumeDecoding == true {
                 contentView.renderingComponent.flush()
@@ -217,13 +218,22 @@ final class StreamPictureInPictureVideoRenderer: UIView, RTCVideoRenderer {
         for track: RTCVideoTrack?,
         on window: UIWindow?
     ) {
-        guard window != nil, let track else { return }
+        guard window != nil else { return }
+
+        staticFrameGenerationCancellable?.cancel()
+        staticVideoSource.isEnabled = track == nil
+        if let track {
+            track.add(self)
+        } else {
+            staticFrameGenerationCancellable = staticVideoSource
+                .renderingPublisher
+                .sink { [weak self] in self?.renderFrame($0) }
+        }
 
         bufferUpdatesCancellable = bufferPublisher
             .receive(on: DispatchQueue.main)
             .sink { [weak self] in self?.process($0) }
 
-        track.add(self)
         logMessage(
             .debug,
             message: "⏳ Frame streaming for Picture-in-Picture started."
diff --git a/Sources/StreamVideoSwiftUI/Utils/StreamRTCStaticVideoSource/StreamRTCStaticVideoSource.swift b/Sources/StreamVideoSwiftUI/Utils/StreamRTCStaticVideoSource/StreamRTCStaticVideoSource.swift
diff --git a/Sources/StreamVideoSwiftUI/Utils/UserAvatar.swift b/Sources/StreamVideoSwiftUI/Utils/UserAvatar.swift
diff --git a/StreamVideo.xcodeproj/project.pbxproj b/StreamVideo.xcodeproj/project.pbxproj

Original file line number	Diff line number	Diff line change
`@@ -18,6 +18,12 @@ public final class SerialActorQueue: Sendable {`
`18`	`18`	/// Initializes a new `SerialActorQueue` instance.
`19`	`19`	`public init() {}`
`20`	`20`
	`21`	`+ private let disposableBag = DisposableBag()`
	`22`	`+`
	`23`	`+ public func cancelAll() {`
	`24`	`+ disposableBag.removeAll()`
	`25`	`+ }`
	`26`	`+`
`21`	`27`	`/// Submits an asynchronous task to be executed serially.`
`22`	`28`	`///`
`23`	`29`	`/// - Parameters:`
`@@ -46,7 +52,7 @@ public final class SerialActorQueue: Sendable {`
`46`	`52`	`lineNumber: line`
`47`	`53`	`)`
`48`	`54`	`}`
`49`		`- }`
	`55`	`+ }.store(in: disposableBag)`
`50`	`56`	`}`
`51`	`57`
`52`	`58`	`/// Submits an asynchronous task to be executed serially and waits for it to complete.`