From f5b0b42f00fa21fa0439ed356dc0c026e5dcdc11 Mon Sep 17 00:00:00 2001 From: jcm Date: Sun, 14 Jan 2024 16:28:11 -0600 Subject: [PATCH] virtual camera, other things --- CaptureSample/CaptureEngine.swift | 8 + CaptureSample/CaptureSample.entitlements | 2 + CaptureSample/Logging.swift | 2 + CaptureSample/RecordCameraStreamSink.swift | 236 ++++++++++++++++ CaptureSample/ReplayBuffer.swift | 139 +++++++--- CaptureSample/ScreenRecorder.swift | 38 ++- .../AppControlsConfigurationView.swift | 25 ++ Record.xcodeproj/project.pbxproj | 8 + .../RecordCameraExtensionProvider.swift | 259 +++++++++++------- .../RecordCameraExtensionSink.swift | 100 +++++++ 10 files changed, 670 insertions(+), 147 deletions(-) create mode 100644 CaptureSample/RecordCameraStreamSink.swift create mode 100644 RecordCameraExtension/RecordCameraExtensionSink.swift diff --git a/CaptureSample/CaptureEngine.swift b/CaptureSample/CaptureEngine.swift index edf48da..27e65cf 100644 --- a/CaptureSample/CaptureEngine.swift +++ b/CaptureSample/CaptureEngine.swift @@ -122,6 +122,9 @@ class CaptureEngineStreamOutput: NSObject, SCStreamOutput, SCStreamDelegate { var dstData: UnsafeMutableRawPointer! private let frameHandlerQueue = DispatchQueue(label: "com.jcm.Record.FrameHandlerQueue") + var sink: RecordCameraStreamSink! = RecordCameraStreamSink() + var sinkInitialized = false + var framesWritten = 0 private let logger = Logger.capture @@ -147,10 +150,15 @@ class CaptureEngineStreamOutput: NSObject, SCStreamOutput, SCStreamDelegate { self.logger.notice("ScreenCaptureKit emitted an invalid frame; skipping it. Timestamp: \(sampleBuffer.presentationTimeStamp.seconds, privacy: .public)") return } + if !self.sinkInitialized { + self.sink.connectToCamera(width: 3456, height: 2234) + self.sinkInitialized = true + } switch outputType { case .screen: if let frame = self.createFrame(for: sampleBuffer) { self.capturedFrameHandler?(frame) + self.sink.enqueue(frame.surface!) } case .audio: if let copy = self.createAudioFrame(for: sampleBuffer) { diff --git a/CaptureSample/CaptureSample.entitlements b/CaptureSample/CaptureSample.entitlements index 07c80a8..81480c6 100644 --- a/CaptureSample/CaptureSample.entitlements +++ b/CaptureSample/CaptureSample.entitlements @@ -10,6 +10,8 @@ $(TeamIdentifierPrefix)com.jcm.Record + com.apple.security.device.camera + com.apple.security.files.user-selected.read-write diff --git a/CaptureSample/Logging.swift b/CaptureSample/Logging.swift index e90649a..7f3b5ce 100644 --- a/CaptureSample/Logging.swift +++ b/CaptureSample/Logging.swift @@ -22,6 +22,8 @@ extension Logger { static let application = Logger(subsystem: subsystem, category: "application") + static let virtualCamera = Logger(subsystem: subsystem, category: "virtualCamera") + func generateLog() async -> TextDocument? { do { let logStore = try OSLogStore(scope: .currentProcessIdentifier) diff --git a/CaptureSample/RecordCameraStreamSink.swift b/CaptureSample/RecordCameraStreamSink.swift new file mode 100644 index 0000000..79c2946 --- /dev/null +++ b/CaptureSample/RecordCameraStreamSink.swift @@ -0,0 +1,236 @@ +import Foundation +import CoreMediaIO +import AVFoundation +import OSLog + + +class RecordCameraStreamSink: NSObject { + + private let logger = Logger.virtualCamera + + var sourceStream: CMIOStreamID? + var sinkStream: CMIOStreamID? + var sinkQueue: CMSimpleQueue? + var cameraName = "RecordCameraExtension (Swift)" + var testProperty = "dog" + + private var needToStream: Bool = false + private var mirrorCamera: Bool = false + private var activating: Bool = false + private var readyToEnqueue = false + private var enqueued = false + private var _videoDescription: CMFormatDescription! + private var _bufferPool: CVPixelBufferPool! + private var _bufferAuxAttributes: NSDictionary! + private var _whiteStripeStartRow: UInt32 = 0 + private var _whiteStripeIsAscending: Bool = false + private var overlayMessage: Bool = false + private var sequenceNumber = 0 + private var timer: Timer? + private var propTimer: Timer? + + func getJustProperty(streamId: CMIOStreamID) -> String? { + let selector = "just".convertedToCMIOObjectPropertySelectorName() + var address = CMIOObjectPropertyAddress(selector, .global, .main) + let exists = CMIOObjectHasProperty(streamId, &address) + if exists { + var dataSize: UInt32 = 0 + var dataUsed: UInt32 = 0 + CMIOObjectGetPropertyDataSize(streamId, &address, 0, nil, &dataSize) + var name: CFString = "" as NSString + CMIOObjectGetPropertyData(streamId, &address, 0, nil, dataSize, &dataUsed, &name); + return name as String + } else { + return nil + } + } + + func setJustProperty(streamId: CMIOStreamID, newValue: String) { + let selector = "just".convertedToCMIOObjectPropertySelectorName() + var address = CMIOObjectPropertyAddress(selector, .global, .main) + let exists = CMIOObjectHasProperty(streamId, &address) + if exists { + var settable: DarwinBoolean = false + CMIOObjectIsPropertySettable(streamId,&address,&settable) + if settable == false { + return + } + var dataSize: UInt32 = 0 + CMIOObjectGetPropertyDataSize(streamId, &address, 0, nil, &dataSize) + var newName: CFString = newValue as NSString + CMIOObjectSetPropertyData(streamId, &address, 0, nil, dataSize, &newName) + } + } + + + func initSink(deviceId: CMIODeviceID, sinkStream: CMIOStreamID, width: Int32, height: Int32) { + let dims = CMVideoDimensions(width: width, height: height) + CMVideoFormatDescriptionCreate( + allocator: kCFAllocatorDefault, + codecType: kCVPixelFormatType_32BGRA, + width: dims.width, height: dims.height, extensions: nil, formatDescriptionOut: &_videoDescription) + + var pixelBufferAttributes: NSDictionary! + pixelBufferAttributes = [ + kCVPixelBufferWidthKey: dims.width, + kCVPixelBufferHeightKey: dims.height, + kCVPixelBufferPixelFormatTypeKey: _videoDescription.mediaSubType, + kCVPixelBufferIOSurfacePropertiesKey: [:] + ] + + CVPixelBufferPoolCreate(kCFAllocatorDefault, nil, pixelBufferAttributes, &_bufferPool) + + let pointerQueue = UnsafeMutablePointer?>.allocate(capacity: 1) + let pointerRef = UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) + let result = CMIOStreamCopyBufferQueue(sinkStream, { + (sinkStream: CMIOStreamID, buf: UnsafeMutableRawPointer?, refcon: UnsafeMutableRawPointer?) in + let sender = Unmanaged.fromOpaque(refcon!).takeUnretainedValue() + sender.readyToEnqueue = true + },pointerRef,pointerQueue) + if result != 0 { + logger.error("error copying buffer queue") + } else { + if let queue = pointerQueue.pointee { + self.sinkQueue = queue.takeUnretainedValue() + } + let resultStart = CMIODeviceStartStream(deviceId, sinkStream) == 0 + if resultStart { + logger.info("virtual camera sink started") + } else { + logger.error("error starting virtual camera sink") + } + } + } + + func getDevice(name: String) -> AVCaptureDevice? { + print("getDevice name=",name) + var devices: [AVCaptureDevice]? + let discoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: [.externalUnknown], + mediaType: .video, + position: .unspecified) + devices = discoverySession.devices + guard let devices = devices else { return nil } + return devices.first { $0.localizedName == name} + } + + func getCMIODevice(uid: String) -> CMIOObjectID? { + var dataSize: UInt32 = 0 + var devices = [CMIOObjectID]() + var dataUsed: UInt32 = 0 + var opa = CMIOObjectPropertyAddress(CMIOObjectPropertySelector(kCMIOHardwarePropertyDevices), .global, .main) + CMIOObjectGetPropertyDataSize(CMIOObjectPropertySelector(kCMIOObjectSystemObject), &opa, 0, nil, &dataSize); + let nDevices = Int(dataSize) / MemoryLayout.size + devices = [CMIOObjectID](repeating: 0, count: Int(nDevices)) + CMIOObjectGetPropertyData(CMIOObjectPropertySelector(kCMIOObjectSystemObject), &opa, 0, nil, dataSize, &dataUsed, &devices); + for deviceObjectID in devices { + opa.mSelector = CMIOObjectPropertySelector(kCMIODevicePropertyDeviceUID) + CMIOObjectGetPropertyDataSize(deviceObjectID, &opa, 0, nil, &dataSize) + var name: CFString = "" as NSString + //CMIOObjectGetPropertyData(deviceObjectID, &opa, 0, nil, UInt32(MemoryLayout.size), &dataSize, &name); + CMIOObjectGetPropertyData(deviceObjectID, &opa, 0, nil, dataSize, &dataUsed, &name); + if String(name) == uid { + return deviceObjectID + } + } + return nil + } + + func getInputStreams(deviceId: CMIODeviceID) -> [CMIOStreamID] { + var dataSize: UInt32 = 0 + var dataUsed: UInt32 = 0 + var opa = CMIOObjectPropertyAddress(CMIOObjectPropertySelector(kCMIODevicePropertyStreams), .global, .main) + CMIOObjectGetPropertyDataSize(deviceId, &opa, 0, nil, &dataSize); + let numberStreams = Int(dataSize) / MemoryLayout.size + var streamIds = [CMIOStreamID](repeating: 0, count: numberStreams) + CMIOObjectGetPropertyData(deviceId, &opa, 0, nil, dataSize, &dataUsed, &streamIds) + return streamIds + } + + func connectToCamera(width: Int32, height: Int32) { + if let device = getDevice(name: "RecordCameraExtension (Swift)"), let deviceObjectId = getCMIODevice(uid: device.uniqueID) { + let streamIds = getInputStreams(deviceId: deviceObjectId) + if streamIds.count == 2 { + sinkStream = streamIds[1] + logger.info("found sink stream") + initSink(deviceId: deviceObjectId, sinkStream: streamIds[1], width: width, height: height) + } + if let firstStream = streamIds.first { + logger.info("found source stream") + sourceStream = firstStream + } + } + } + + func enqueue(_ image: IOSurfaceRef) { + guard CMSimpleQueueGetCount(sinkQueue!) < CMSimpleQueueGetCapacity(sinkQueue!) else { + print("error enqueuing") + return + } + var err: OSStatus = 0 + var pixelBuffer: Unmanaged? + CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, image, self._bufferAuxAttributes, &pixelBuffer) + if let pixelBuffer = pixelBuffer { + + var sbuf: CMSampleBuffer! + var timingInfo = CMSampleTimingInfo() + timingInfo.presentationTimeStamp = CMClockGetTime(CMClockGetHostTimeClock()) + err = CMSampleBufferCreateForImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: pixelBuffer.takeRetainedValue(), dataReady: true, makeDataReadyCallback: nil, refcon: nil, formatDescription: self._videoDescription, sampleTiming: &timingInfo, sampleBufferOut: &sbuf) + if err == 0 { + if let sbuf = sbuf { + let pointerRef = UnsafeMutableRawPointer(Unmanaged.passRetained(sbuf).toOpaque()) + CMSimpleQueueEnqueue(self.sinkQueue!, element: pointerRef) + } + } + } else { + print("error getting pixel buffer") + } + } + +} + +extension String { + func convertedToCMIOObjectPropertySelectorName() -> CMIOObjectPropertySelector { + let noName: CMIOObjectPropertySelector = 0 + if count == MemoryLayout.size { + return data(using: .utf8, allowLossyConversion: false)?.withUnsafeBytes { propertySelector in + propertySelector.load(as: CMIOObjectPropertySelector.self).byteSwapped + } ?? noName + } else { + return noName + } + } +} + +public extension CMIOObjectPropertyAddress { + init(_ selector: CMIOObjectPropertySelector, + _ scope: CMIOObjectPropertyScope = .anyScope, + _ element: CMIOObjectPropertyElement = .anyElement) { + self.init(mSelector: selector, mScope: scope, mElement: element) + } +} + +public extension CMIOObjectPropertyScope { + /// The CMIOObjectPropertyScope for properties that apply to the object as a whole. + /// All CMIOObjects have a global scope and for some it is their only scope. + static let global = CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal) + + /// The wildcard value for CMIOObjectPropertyScopes. + static let anyScope = CMIOObjectPropertyScope(kCMIOObjectPropertyScopeWildcard) + + /// The CMIOObjectPropertyScope for properties that apply to the input signal paths of the CMIODevice. + static let deviceInput = CMIOObjectPropertyScope(kCMIODevicePropertyScopeInput) + + /// The CMIOObjectPropertyScope for properties that apply to the output signal paths of the CMIODevice. + static let deviceOutput = CMIOObjectPropertyScope(kCMIODevicePropertyScopeOutput) + + /// The CMIOObjectPropertyScope for properties that apply to the play through signal paths of the CMIODevice. + static let devicePlayThrough = CMIOObjectPropertyScope(kCMIODevicePropertyScopePlayThrough) +} + +public extension CMIOObjectPropertyElement { + /// The CMIOObjectPropertyElement value for properties that apply to the master element or to the entire scope. + //static let master = CMIOObjectPropertyElement(kCMIOObjectPropertyElementMaster) + static let main = CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain) + /// The wildcard value for CMIOObjectPropertyElements. + static let anyElement = CMIOObjectPropertyElement(kCMIOObjectPropertyElementWildcard) +} diff --git a/CaptureSample/ReplayBuffer.swift b/CaptureSample/ReplayBuffer.swift index b37a846..72ae677 100644 --- a/CaptureSample/ReplayBuffer.swift +++ b/CaptureSample/ReplayBuffer.swift @@ -137,49 +137,102 @@ extension CMSampleBuffer { } func deepCopy() -> CMSampleBuffer? { - guard let formatDesc = CMSampleBufferGetFormatDescription(self), - let data = try? self.dataBuffer?.dataBytes() else { - return nil - } - let nFrames = CMSampleBufferGetNumSamples(self) - let pts = CMSampleBufferGetPresentationTimeStamp(self) - let dataBuffer = data.withUnsafeBytes { (buffer) -> CMBlockBuffer? in - var blockBuffer: CMBlockBuffer? - let length: Int = data.count - guard CMBlockBufferCreateWithMemoryBlock( - allocator: kCFAllocatorDefault, - memoryBlock: nil, - blockLength: length, - blockAllocator: nil, - customBlockSource: nil, - offsetToData: 0, - dataLength: length, - flags: 0, - blockBufferOut: &blockBuffer) == noErr else { - return nil - } - guard CMBlockBufferReplaceDataBytes( - with: buffer.baseAddress!, - blockBuffer: blockBuffer!, - offsetIntoDestination: 0, - dataLength: length) == noErr else { - return nil - } - return blockBuffer - } - guard let dataBuffer = dataBuffer else { - return nil - } - var newSampleBuffer: CMSampleBuffer? - CMAudioSampleBufferCreateReadyWithPacketDescriptions( + guard let formatDesc = CMSampleBufferGetFormatDescription(self), + let data = try? self.dataBuffer?.dataBytes() else { + return nil + } + let nFrames = CMSampleBufferGetNumSamples(self) + let pts = CMSampleBufferGetPresentationTimeStamp(self) + let dataBuffer = data.withUnsafeBytes { (buffer) -> CMBlockBuffer? in + var blockBuffer: CMBlockBuffer? + let length: Int = data.count + guard CMBlockBufferCreateWithMemoryBlock( allocator: kCFAllocatorDefault, - dataBuffer: dataBuffer, - formatDescription: formatDesc, - sampleCount: nFrames, - presentationTimeStamp: pts, - packetDescriptions: nil, - sampleBufferOut: &newSampleBuffer - ) - return newSampleBuffer + memoryBlock: nil, + blockLength: length, + blockAllocator: nil, + customBlockSource: nil, + offsetToData: 0, + dataLength: length, + flags: 0, + blockBufferOut: &blockBuffer) == noErr else { + return nil + } + guard CMBlockBufferReplaceDataBytes( + with: buffer.baseAddress!, + blockBuffer: blockBuffer!, + offsetIntoDestination: 0, + dataLength: length) == noErr else { + return nil + } + return blockBuffer + } + guard let dataBuffer = dataBuffer else { + return nil } + var newSampleBuffer: CMSampleBuffer? + CMAudioSampleBufferCreateReadyWithPacketDescriptions( + allocator: kCFAllocatorDefault, + dataBuffer: dataBuffer, + formatDescription: formatDesc, + sampleCount: nFrames, + presentationTimeStamp: pts, + packetDescriptions: nil, + sampleBufferOut: &newSampleBuffer + ) + return newSampleBuffer + } +} + +extension CVPixelBuffer { + func copy() -> CVPixelBuffer { + precondition(CFGetTypeID(self) == CVPixelBufferGetTypeID(), "copy() cannot be called on a non-CVPixelBuffer") + + let ioSurfaceProps = [ + "IOSurfaceOpenGLESFBOCompatibility": true as CFBoolean, + "IOSurfaceOpenGLESTextureCompatibility": true as CFBoolean, + "IOSurfaceCoreAnimationCompatibility": true as CFBoolean + ] as CFDictionary + + let options = [ + String(kCVPixelBufferMetalCompatibilityKey): true as CFBoolean, + String(kCVPixelBufferIOSurfacePropertiesKey): ioSurfaceProps + ] as CFDictionary + + var _copy : CVPixelBuffer? + CVPixelBufferCreate( + nil, + CVPixelBufferGetWidth(self), + CVPixelBufferGetHeight(self), + CVPixelBufferGetPixelFormatType(self), + options, + &_copy) + + guard let copy = _copy else { fatalError() } + + CVBufferPropagateAttachments(self as CVBuffer, copy as CVBuffer) + + CVPixelBufferLockBaseAddress(self, CVPixelBufferLockFlags.readOnly) + CVPixelBufferLockBaseAddress(copy, CVPixelBufferLockFlags(rawValue: 0)) + + let copyBaseAddress = CVPixelBufferGetBaseAddress(copy) + let currBaseAddress = CVPixelBufferGetBaseAddress(self) + + memcpy(copyBaseAddress, currBaseAddress, CVPixelBufferGetDataSize(self)) + + CVPixelBufferUnlockBaseAddress(copy, CVPixelBufferLockFlags(rawValue: 0)) + CVPixelBufferUnlockBaseAddress(self, CVPixelBufferLockFlags.readOnly) + + // let's make sure they have the same average color +// let originalImage = CIImage(cvPixelBuffer: self) +// let copiedImage = CIImage(cvPixelBuffer: copy) +// +// let averageColorOriginal = originalImage.averageColour() +// let averageColorCopy = copiedImage.averageColour() +// +// assert(averageColorCopy == averageColorOriginal) +// debugPrint("average frame color: \(averageColorCopy)") + + return copy + } } diff --git a/CaptureSample/ScreenRecorder.swift b/CaptureSample/ScreenRecorder.swift index 33de113..2caa181 100644 --- a/CaptureSample/ScreenRecorder.swift +++ b/CaptureSample/ScreenRecorder.swift @@ -22,7 +22,7 @@ class AudioLevelsProvider: ObservableObject { @MainActor class ScreenRecorder: ObservableObject { - + var sink: RecordCameraStreamSink! private var extensionActivated = false @@ -548,15 +548,33 @@ class ScreenRecorder: ObservableObject { if self.showsEncodePreview { self.updateEncodePreview() } - if !self.extensionActivated { - let identifier = "com.jcm.Record.RecordCameraExtension" - - - // Submit an activation request. - let activationRequest = OSSystemExtensionRequest.activationRequest(forExtensionWithIdentifier: identifier, queue: .main) - activationRequest.delegate = self.requestDelegate - OSSystemExtensionManager.shared.submitRequest(activationRequest) - } + } + + func uninstallExtension() { + let identifier = "com.jcm.Record.RecordCameraExtension" + + + // Submit an activation request. + /*let activationRequest = OSSystemExtensionRequest.activationRequest(forExtensionWithIdentifier: identifier, queue: .main) + activationRequest.delegate = self.requestDelegate + OSSystemExtensionManager.shared.submitRequest(activationRequest)*/ + let deactivationRequest = OSSystemExtensionRequest.deactivationRequest(forExtensionWithIdentifier: identifier, queue: .main) + deactivationRequest.delegate = self.requestDelegate + OSSystemExtensionManager.shared.submitRequest(deactivationRequest) + } + + func installExtension() { + let identifier = "com.jcm.Record.RecordCameraExtension" + + + // Submit an activation request. + let activationRequest = OSSystemExtensionRequest.activationRequest(forExtensionWithIdentifier: identifier, queue: .main) + activationRequest.delegate = self.requestDelegate + OSSystemExtensionManager.shared.submitRequest(activationRequest) + } + + func testSetProperty() { + print("poop") } /// - Tag: UpdateFilter diff --git a/CaptureSample/Views/Configuration View/AppControlsConfigurationView.swift b/CaptureSample/Views/Configuration View/AppControlsConfigurationView.swift index 71fb0b0..94c83e3 100644 --- a/CaptureSample/Views/Configuration View/AppControlsConfigurationView.swift +++ b/CaptureSample/Views/Configuration View/AppControlsConfigurationView.swift @@ -74,5 +74,30 @@ struct AppControlsConfigurationView: View { } .frame(maxWidth: .infinity) .padding(EdgeInsets(top: 0, leading: 0, bottom: 15, trailing: 0)) + HStack { + Button { + Task { await screenRecorder.uninstallExtension() } + } label: { + Text("Remove Extension") + } + .controlSize(.large) + .buttonStyle(.borderedProminent) + Button { + Task { screenRecorder.installExtension() } + } label: { + Text("Install Extension") + } + .controlSize(.large) + .buttonStyle(.borderedProminent) + Button { + Task { screenRecorder.testSetProperty() } + } label: { + Text("Test Set Property") + } + .controlSize(.large) + .buttonStyle(.borderedProminent) + } + .frame(maxWidth: .infinity) + .padding(EdgeInsets(top: 0, leading: 0, bottom: 15, trailing: 0)) } } diff --git a/Record.xcodeproj/project.pbxproj b/Record.xcodeproj/project.pbxproj index 0df58e8..3fc9cee 100644 --- a/Record.xcodeproj/project.pbxproj +++ b/Record.xcodeproj/project.pbxproj @@ -33,6 +33,8 @@ 0D6933012B34B1850019368E /* RecordCameraExtensionProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0D6933002B34B1850019368E /* RecordCameraExtensionProvider.swift */; }; 0D6933032B34B1850019368E /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0D6933022B34B1850019368E /* main.swift */; }; 0D6933082B34B1850019368E /* com.jcm.Record.RecordCameraExtension.systemextension in Embed System Extensions */ = {isa = PBXBuildFile; fileRef = 0D6932FE2B34B1850019368E /* com.jcm.Record.RecordCameraExtension.systemextension */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + 0D6933B02B35322B0019368E /* RecordCameraExtensionSink.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0D6933AF2B35322B0019368E /* RecordCameraExtensionSink.swift */; }; + 0D6933B22B3688FD0019368E /* RecordCameraStreamSink.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0D6933B12B3688FD0019368E /* RecordCameraStreamSink.swift */; }; 0DE5C82F2A95EDC60054AC23 /* PickerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0DE5C82E2A95EDC60054AC23 /* PickerView.swift */; }; C470F0812811C5CB00D29309 /* ScreenRecorder.swift in Sources */ = {isa = PBXBuildFile; fileRef = C470F0802811C5CB00D29309 /* ScreenRecorder.swift */; }; C471DFFB2809F440001D24C9 /* PowerMeter.swift in Sources */ = {isa = PBXBuildFile; fileRef = C471DFF92809F440001D24C9 /* PowerMeter.swift */; }; @@ -101,6 +103,8 @@ 0D6933022B34B1850019368E /* main.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = main.swift; sourceTree = ""; }; 0D6933042B34B1850019368E /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 0D6933052B34B1850019368E /* RecordCameraExtension.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = RecordCameraExtension.entitlements; sourceTree = ""; }; + 0D6933AF2B35322B0019368E /* RecordCameraExtensionSink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecordCameraExtensionSink.swift; sourceTree = ""; }; + 0D6933B12B3688FD0019368E /* RecordCameraStreamSink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecordCameraStreamSink.swift; sourceTree = ""; }; 0DE5C82E2A95EDC60054AC23 /* PickerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PickerView.swift; sourceTree = ""; }; 0DF11FDD2A6ECBA500B45306 /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.markdown; }; 7C6C99F1D4B6E3EBA3A7B7DF /* LICENSE.txt */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text; path = LICENSE.txt; sourceTree = ""; }; @@ -177,6 +181,7 @@ isa = PBXGroup; children = ( 0D6933002B34B1850019368E /* RecordCameraExtensionProvider.swift */, + 0D6933AF2B35322B0019368E /* RecordCameraExtensionSink.swift */, 0D6933022B34B1850019368E /* main.swift */, 0D6933042B34B1850019368E /* Info.plist */, 0D6933052B34B1850019368E /* RecordCameraExtension.entitlements */, @@ -232,6 +237,7 @@ C4B0DAA9276BA4460015082A /* CaptureSample */ = { isa = PBXGroup; children = ( + 0D6933B12B3688FD0019368E /* RecordCameraStreamSink.swift */, C470F0802811C5CB00D29309 /* ScreenRecorder.swift */, 0D48753F2A81A20800F6D832 /* Enums.swift */, 0D1326772A85612B00B1E886 /* CVPixelBufferHelpers.swift */, @@ -377,6 +383,7 @@ buildActionMask = 2147483647; files = ( 0D6933032B34B1850019368E /* main.swift in Sources */, + 0D6933B02B35322B0019368E /* RecordCameraExtensionSink.swift in Sources */, 0D6933012B34B1850019368E /* RecordCameraExtensionProvider.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -414,6 +421,7 @@ 0D152E9F2AB9FF2700FEB9CE /* HotkeysPreferencesView.swift in Sources */, 0D164AA02A97E894003F2F7E /* OutputConfigurationView.swift in Sources */, 0D164AAA2A97EC32003F2F7E /* ReplayBufferTabItem.swift in Sources */, + 0D6933B22B3688FD0019368E /* RecordCameraStreamSink.swift in Sources */, 0D368B1F2A6CED2F00DA288D /* EncoderOptions.swift in Sources */, 0D3065CC2A9466A600247474 /* Renderer.swift in Sources */, 0D164AA82A97EBDB003F2F7E /* KeyframesTabItem.swift in Sources */, diff --git a/RecordCameraExtension/RecordCameraExtensionProvider.swift b/RecordCameraExtension/RecordCameraExtensionProvider.swift index c897550..de6299e 100644 --- a/RecordCameraExtension/RecordCameraExtensionProvider.swift +++ b/RecordCameraExtension/RecordCameraExtensionProvider.swift @@ -1,28 +1,36 @@ -// -// RecordCameraExtensionProvider.swift -// RecordCameraExtension -// -// Created by John Moody on 12/21/23. -// Copyright © 2023 jcm. All rights reserved. -// - import Foundation import CoreMediaIO import IOKit.audio import os.log - -let kWhiteStripeHeight: Int = 10 -let kFrameRate: Int = 60 +import AppKit // MARK: - +let customExtensionPropertyTest: CMIOExtensionProperty = CMIOExtensionProperty(rawValue: "4cc_test_glob_0000") +let kFrameRate: Int = 60 + class RecordCameraExtensionDeviceSource: NSObject, CMIOExtensionDeviceSource { private(set) var device: CMIOExtensionDevice! private var _streamSource: RecordCameraExtensionStreamSource! - - private var _streamingCounter: UInt32 = 0 + public var _streamSink: RecordCameraExtensionStreamSink! + private var _streamingCounter: UInt32 = 0 + private var _streamingSinkCounter: UInt32 = 0 + + let kWhiteStripeHeight: Int = 10 + + var lastMessage = "" + + var textFontAttributes: [NSAttributedString.Key : Any]! + + let textColor = NSColor.white + let fontSize = 24.0 + var textFont: NSFont! + + func myStreamingCounter() -> String { + return "sc=\(_streamingCounter)" + } private var _timer: DispatchSourceTimer? @@ -37,14 +45,22 @@ class RecordCameraExtensionDeviceSource: NSObject, CMIOExtensionDeviceSource { private var _whiteStripeStartRow: UInt32 = 0 private var _whiteStripeIsAscending: Bool = false + + var stupidCount = 0 init(localizedName: String) { super.init() - let deviceID = UUID(uuidString: "7626645E-4425-469E-9D8B-97E0FA59AC75")! // replace this with your device UUID + textFont = NSFont.systemFont(ofSize: fontSize) + textFontAttributes = [ + NSAttributedString.Key.font: textFont, + NSAttributedString.Key.foregroundColor: textColor, + NSAttributedString.Key.paragraphStyle: NSTextAlignment.center + ] + let deviceID = UUID(uuidString: "4B8051B1-26DF-4958-8354-F01DCB1DB02D")! // replace this with your device UUID self.device = CMIOExtensionDevice(localizedName: localizedName, deviceID: deviceID, legacyDeviceID: nil, source: self) - let dims = CMVideoDimensions(width: 1920, height: 1080) + let dims = CMVideoDimensions(width: 3456, height: 2234) CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault, codecType: kCVPixelFormatType_32BGRA, width: dims.width, height: dims.height, extensions: nil, formatDescriptionOut: &_videoDescription) let pixelBufferAttributes: NSDictionary = [ @@ -58,10 +74,13 @@ class RecordCameraExtensionDeviceSource: NSObject, CMIOExtensionDeviceSource { let videoStreamFormat = CMIOExtensionStreamFormat.init(formatDescription: _videoDescription, maxFrameDuration: CMTime(value: 1, timescale: Int32(kFrameRate)), minFrameDuration: CMTime(value: 1, timescale: Int32(kFrameRate)), validFrameDurations: nil) _bufferAuxAttributes = [kCVPixelBufferPoolAllocationThresholdKey: 5] - let videoID = UUID(uuidString: "A8D7B8AA-65AD-4D21-9C42-66480DBFA8E1")! // replace this with your video UUID + let videoID = UUID(uuidString: "5A36AE62-37CF-4D89-AF81-F9E03FC15907")! // replace this with your video UUID _streamSource = RecordCameraExtensionStreamSource(localizedName: "RecordCameraExtension.Video", streamID: videoID, streamFormat: videoStreamFormat, device: device) + let videoSinkID = UUID(uuidString: "26CCE162-2DB0-4C71-A92C-4F81437BD883")! + _streamSink = RecordCameraExtensionStreamSink(localizedName: "RecordCameraExtension.Video.Sink", streamID: videoSinkID, streamFormat: videoStreamFormat, device: device) do { try device.addStream(_streamSource.stream) + try device.addStream(_streamSink.stream) } catch let error { fatalError("Failed to add stream: \(error.localizedDescription)") } @@ -102,6 +121,9 @@ class RecordCameraExtensionDeviceSource: NSObject, CMIOExtensionDeviceSource { _timer!.schedule(deadline: .now(), repeating: 1.0 / Double(kFrameRate), leeway: .seconds(0)) _timer!.setEventHandler { + if self.sinkStarted { + return + } var err: OSStatus = 0 let now = CMClockGetTime(CMClockGetHostTimeClock()) @@ -129,10 +151,10 @@ class RecordCameraExtensionDeviceSource: NSObject, CMIOExtensionDeviceSource { } else { self._whiteStripeStartRow = whiteStripeStartRow + 1 - self._whiteStripeIsAscending = self._whiteStripeStartRow >= (height - kWhiteStripeHeight) + self._whiteStripeIsAscending = self._whiteStripeStartRow >= (height - self.kWhiteStripeHeight) } bufferPtr += rowBytes * Int(whiteStripeStartRow) - for _ in 0.. 0 { + os_log("sending boofer") + self._streamSource.stream.send(sbuf!, discontinuity: [], hostTimeInNanoseconds: UInt64(sbuf!.presentationTimeStamp.seconds * Double(NSEC_PER_SEC))) + } + self._streamSink.stream.notifyScheduledOutputChanged(output) + } + if err != nil { + os_log("LOGGING AN ERROR POOPY") + os_log("\(err!.localizedDescription)") + } + self.consumeBuffer(client) + } + } + + func startStreamingSink(client: CMIOExtensionClient) { + _streamingSinkCounter += 1 + self.sinkStarted = true + consumeBuffer(client) + } + + func stopStreamingSink() { + self.sinkStarted = false + if _streamingSinkCounter > 1 { + _streamingSinkCounter -= 1 + } + else { + _streamingSinkCounter = 0 + } } } @@ -182,81 +241,93 @@ class RecordCameraExtensionDeviceSource: NSObject, CMIOExtensionDeviceSource { class RecordCameraExtensionStreamSource: NSObject, CMIOExtensionStreamSource { - private(set) var stream: CMIOExtensionStream! - - let device: CMIOExtensionDevice - - private let _streamFormat: CMIOExtensionStreamFormat - - init(localizedName: String, streamID: UUID, streamFormat: CMIOExtensionStreamFormat, device: CMIOExtensionDevice) { - - self.device = device - self._streamFormat = streamFormat - super.init() - self.stream = CMIOExtensionStream(localizedName: localizedName, streamID: streamID, direction: .source, clockType: .hostTime, source: self) - } - - var formats: [CMIOExtensionStreamFormat] { - - return [_streamFormat] - } - - var activeFormatIndex: Int = 0 { - - didSet { - if activeFormatIndex >= 1 { - os_log(.error, "Invalid index") - } - } - } - - var availableProperties: Set { - - return [.streamActiveFormatIndex, .streamFrameDuration] - } - - func streamProperties(forProperties properties: Set) throws -> CMIOExtensionStreamProperties { - - let streamProperties = CMIOExtensionStreamProperties(dictionary: [:]) - if properties.contains(.streamActiveFormatIndex) { - streamProperties.activeFormatIndex = 0 - } - if properties.contains(.streamFrameDuration) { - let frameDuration = CMTime(value: 1, timescale: Int32(kFrameRate)) - streamProperties.frameDuration = frameDuration - } - - return streamProperties - } - - func setStreamProperties(_ streamProperties: CMIOExtensionStreamProperties) throws { - - if let activeFormatIndex = streamProperties.activeFormatIndex { - self.activeFormatIndex = activeFormatIndex - } - } - - func authorizedToStartStream(for client: CMIOExtensionClient) -> Bool { - - // An opportunity to inspect the client info and decide if it should be allowed to start the stream. - return true - } - - func startStream() throws { - - guard let deviceSource = device.source as? RecordCameraExtensionDeviceSource else { - fatalError("Unexpected source type \(String(describing: device.source))") - } - deviceSource.startStreaming() - } - - func stopStream() throws { - - guard let deviceSource = device.source as? RecordCameraExtensionDeviceSource else { - fatalError("Unexpected source type \(String(describing: device.source))") - } - deviceSource.stopStreaming() - } + private(set) var stream: CMIOExtensionStream! + + let device: CMIOExtensionDevice + //public var nConnectedClients = 0 + private let _streamFormat: CMIOExtensionStreamFormat + + init(localizedName: String, streamID: UUID, streamFormat: CMIOExtensionStreamFormat, device: CMIOExtensionDevice) { + + self.device = device + self._streamFormat = streamFormat + super.init() + self.stream = CMIOExtensionStream(localizedName: localizedName, streamID: streamID, direction: .source, clockType: .hostTime, source: self) + } + + var formats: [CMIOExtensionStreamFormat] { + + return [_streamFormat] + } + + var activeFormatIndex: Int = 0 { + + didSet { + if activeFormatIndex >= 1 { + os_log(.error, "Invalid index") + } + } + } + + var availableProperties: Set { + + return [.streamActiveFormatIndex, .streamFrameDuration, customExtensionPropertyTest] + } + + public var test: String = "dog" + var count = 0 + + func streamProperties(forProperties properties: Set) throws -> CMIOExtensionStreamProperties { + let streamProperties = CMIOExtensionStreamProperties(dictionary: [:]) + if properties.contains(.streamActiveFormatIndex) { + streamProperties.activeFormatIndex = 0 + } + if properties.contains(.streamFrameDuration) { + let frameDuration = CMTime(value: 1, timescale: Int32(kFrameRate)) + streamProperties.frameDuration = frameDuration + } + if properties.contains(customExtensionPropertyTest) { + streamProperties.setPropertyState(CMIOExtensionPropertyState(value: self.test as NSString), forProperty: customExtensionPropertyTest) + + } + return streamProperties + } + + func setStreamProperties(_ streamProperties: CMIOExtensionStreamProperties) throws { + + if let activeFormatIndex = streamProperties.activeFormatIndex { + self.activeFormatIndex = activeFormatIndex + } + + if let state = streamProperties.propertiesDictionary[customExtensionPropertyTest] { + if let newValue = state.value as? String { + self.test = newValue + os_log("test is \(self.test, privacy: .public)") + } + } + } + + func authorizedToStartStream(for client: CMIOExtensionClient) -> Bool { + + // An opportunity to inspect the client info and decide if it should be allowed to start the stream. + return true + } + + func startStream() throws { + + guard let deviceSource = device.source as? RecordCameraExtensionDeviceSource else { + fatalError("Unexpected source type \(String(describing: device.source))") + } + deviceSource.startStreaming() + } + + func stopStream() throws { + + guard let deviceSource = device.source as? RecordCameraExtensionDeviceSource else { + fatalError("Unexpected source type \(String(describing: device.source))") + } + deviceSource.stopStreaming() + } } // MARK: - diff --git a/RecordCameraExtension/RecordCameraExtensionSink.swift b/RecordCameraExtension/RecordCameraExtensionSink.swift new file mode 100644 index 0000000..a2a6cb4 --- /dev/null +++ b/RecordCameraExtension/RecordCameraExtensionSink.swift @@ -0,0 +1,100 @@ +// +// RecordCameraExtensionSink.swift +// RecordCameraExtension +// +// Created by John Moody on 12/21/23. +// Copyright © 2023 jcm. All rights reserved. +// + +import CoreMediaIO +import Foundation +import os.log + +class RecordCameraExtensionStreamSink: NSObject, CMIOExtensionStreamSource { + + private(set) var stream: CMIOExtensionStream! + + let device: CMIOExtensionDevice + private let _streamFormat: CMIOExtensionStreamFormat + + init(localizedName: String, streamID: UUID, streamFormat: CMIOExtensionStreamFormat, device: CMIOExtensionDevice) { + + self.device = device + self._streamFormat = streamFormat + super.init() + self.stream = CMIOExtensionStream(localizedName: localizedName, streamID: streamID, direction: .sink, clockType: .hostTime, source: self) + } + + var formats: [CMIOExtensionStreamFormat] { + + return [_streamFormat] + } + + var activeFormatIndex: Int = 0 { + + didSet { + if activeFormatIndex >= 1 { + os_log(.error, "Invalid index") + } + } + } + + var availableProperties: Set { + + return [.streamActiveFormatIndex, .streamFrameDuration, .streamSinkBufferQueueSize, .streamSinkBuffersRequiredForStartup, .streamSinkBufferUnderrunCount, .streamSinkEndOfData] + } + + var client: CMIOExtensionClient? + func streamProperties(forProperties properties: Set) throws -> CMIOExtensionStreamProperties { + let streamProperties = CMIOExtensionStreamProperties(dictionary: [:]) + if properties.contains(.streamActiveFormatIndex) { + streamProperties.activeFormatIndex = 0 + } + if properties.contains(.streamFrameDuration) { + let frameDuration = CMTime(value: 1, timescale: Int32(kFrameRate)) + streamProperties.frameDuration = frameDuration + } + if properties.contains(.streamSinkBufferQueueSize) { + streamProperties.sinkBufferQueueSize = 1 + } + if properties.contains(.streamSinkBuffersRequiredForStartup) { + streamProperties.sinkBuffersRequiredForStartup = 1 + } + return streamProperties + } + + func setStreamProperties(_ streamProperties: CMIOExtensionStreamProperties) throws { + + if let activeFormatIndex = streamProperties.activeFormatIndex { + self.activeFormatIndex = activeFormatIndex + } + + } + + func authorizedToStartStream(for client: CMIOExtensionClient) -> Bool { + + // An opportunity to inspect the client info and decide if it should be allowed to start the stream. + self.client = client + return true + } + + func startStream() throws { + + guard let deviceSource = device.source as? RecordCameraExtensionDeviceSource else { + fatalError("Unexpected source type \(String(describing: device.source))") + } + if let client = client { + os_log("STARTING STREAMING SINK POOPYPOOPYPOO") + deviceSource.startStreamingSink(client: client) + } + os_log("UHHHHH POOPY") + } + + func stopStream() throws { + + guard let deviceSource = device.source as? RecordCameraExtensionDeviceSource else { + fatalError("Unexpected source type \(String(describing: device.source))") + } + deviceSource.stopStreamingSink() + } +}