Reduce iOS video enc/dec redundant code, add more H.265 QP parsing logic

sdk/android/src/jni/video_decoder_wrapper.cc

@@ -257,6 +257,17 @@ absl::optional<uint8_t> VideoDecoderWrapper::ParseQP(
       }
       break;
     }
+#ifndef DISABLE_H265
+    case kVideoCodecH265: {
+      h265_bitstream_parser_.ParseBitstream(input_image.data(),
+                                            input_image.size());
+      int qp_int;
+      if (h265_bitstream_parser_.GetLastSliceQp(&qp_int)) {
+        qp = qp_int;
+      }
+      break;
+    }
+#endif
     default:
       break;  // Default is to not provide QP.
   }

sdk/android/src/jni/video_decoder_wrapper.h

@@ -18,6 +18,9 @@
 
 #include "api/video_codecs/video_decoder.h"
 #include "common_video/h264/h264_bitstream_parser.h"
+#ifndef DISABLE_H265
+#include "common_video/h265/h265_bitstream_parser.h"
+#endif
 #include "rtc_base/race_checker.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/thread_checker.h"
@@ -100,6 +103,10 @@ class VideoDecoderWrapper : public VideoDecoder {
   bool initialized_ RTC_GUARDED_BY(decoder_thread_checker_);
   H264BitstreamParser h264_bitstream_parser_
       RTC_GUARDED_BY(decoder_thread_checker_);
+#ifndef DISABLE_H265
+  H265BitstreamParser h265_bitstream_parser_
+      RTC_GUARDED_BY(decoder_thread_checker_);
+#endif
 
   DecodedImageCallback* callback_ RTC_GUARDED_BY(callback_race_checker_);
 

sdk/objc/base/RTCVideoDecoder.h

@@ -19,7 +19,7 @@
 NS_ASSUME_NONNULL_BEGIN
 
 /** Callback block for decoder. */
-typedef void (^RTCVideoDecoderCallback)(RTC_OBJC_TYPE(RTCVideoFrame) * frame);
+typedef void (^RTCVideoDecoderCallback)(RTC_OBJC_TYPE(RTCVideoFrame) * frame, int32_t qp);
 
 /** Protocol for decoder implementations. */
 RTC_OBJC_EXPORT

sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m

@@ -53,7 +53,19 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory)
 #endif
 
 #if !defined(DISABLE_H265)
-  RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name];
+  if (@available(iOS 11, *)) {
+    if ([RTCVideoDecoderH265 supported]) {
+      return @[
+        constrainedHighInfo,
+        constrainedBaselineInfo,
+        vp8Info,
+#if defined(RTC_ENABLE_VP9)
+        vp9Info,
+#endif
+        [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH265Name],
+      ];
+    }
+  }
 #endif
 
   return @[
@@ -62,16 +74,13 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory)
     vp8Info,
 #if defined(RTC_ENABLE_VP9)
     vp9Info,
-#endif
-#if !defined(DISABLE_H265)
-    h265Info,
 #endif
   ];
 }
 
 - (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)createDecoder:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info {
   if ([info.name isEqualToString:kRTCVideoCodecH264Name]) {
-    return [[RTC_OBJC_TYPE(RTCVideoDecoderH264) alloc] init];
+    return [[RTC_OBJC_TYPE(RTCVideoDecoderH264) alloc] initWithCodecInfo:info];
   } else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) {
     return [RTC_OBJC_TYPE(RTCVideoDecoderVP8) vp8Decoder];
 #if defined(RTC_ENABLE_VP9)
@@ -81,7 +90,7 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory)
 #if !defined(DISABLE_H265)
   } else if (@available(iOS 11, *)) {
     if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
-      return [[RTCVideoDecoderH265 alloc] init];
+      return [[RTC_OBJC_TYPE(RTCVideoDecoderH265) alloc] initWithCodecInfo:info];
     }
 #endif
   }

sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m

@@ -55,7 +55,19 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory)
 #endif
 
 #if !defined(DISABLE_H265)
-  RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name];
+  if (@available(iOS 11, *)) {
+    if ([RTCVideoEncoderH265 supported]) {
+      return @[
+        constrainedHighInfo,
+        constrainedBaselineInfo,
+        vp8Info,
+#if defined(RTC_ENABLE_VP9)
+        vp9Info,
+#endif
+        [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH265Name],
+      ];
+    }
+  }
 #endif
 
   return @[
@@ -64,9 +76,6 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory)
     vp8Info,
 #if defined(RTC_ENABLE_VP9)
     vp9Info,
-#endif
-#if !defined(DISABLE_H265)
-    h265Info,
 #endif
   ];
 }
@@ -83,7 +92,7 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory)
 #if !defined(DISABLE_H265)
   } else if (@available(iOS 11, *)) {
     if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
-      return [[RTCVideoEncoderH265 alloc] initWithCodecInfo:info];
+      return [[RTC_OBJC_TYPE(RTCVideoEncoderH265) alloc] initWithCodecInfo:info];
     }
 #endif
   }

sdk/objc/components/video_codec/RTCVideoDecoderH264.h

@@ -12,7 +12,11 @@
 
 #import "RTCMacros.h"
 #import "RTCVideoDecoder.h"
+#import "RTCVideoCodecInfo.h"
 
 RTC_OBJC_EXPORT
 @interface RTC_OBJC_TYPE (RTCVideoDecoderH264) : NSObject <RTC_OBJC_TYPE(RTCVideoDecoder)>
+
+- (instancetype)initWithCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)codecInfo;
+
 @end

sdk/objc/components/video_codec/RTCVideoDecoderH264.mm

@@ -19,10 +19,16 @@
 #import "helpers.h"
 #import "helpers/scoped_cftyperef.h"
 
+#import "RTCH264ProfileLevelId.h"
+
 #if defined(WEBRTC_IOS)
 #import "helpers/UIDevice+RTCDevice.h"
 #endif
 
+#include "common_video/h264/h264_bitstream_parser.h"
+#if !defined(DISABLE_H265)
+#include "common_video/h265/h265_bitstream_parser.h"
+#endif
 #include "modules/video_coding/include/video_error_codes.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
@@ -32,9 +38,10 @@
 // Struct that we pass to the decoder per frame to decode. We receive it again
 // in the decoder callback.
 struct RTCFrameDecodeParams {
-  RTCFrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts) : callback(cb), timestamp(ts) {}
+  RTCFrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts, int32_t qp) : callback(cb), timestamp(ts), qp(qp) {}
   RTCVideoDecoderCallback callback;
   int64_t timestamp;
+  int32_t qp;
 };
 
 @interface RTC_OBJC_TYPE (RTCVideoDecoderH264)
@@ -67,7 +74,7 @@ void decompressionOutputCallback(void *decoderRef,
             rotation:RTCVideoRotation_0
          timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec];
   decodedFrame.timeStamp = decodeParams->timestamp;
-  decodeParams->callback(decodedFrame);
+  decodeParams->callback(decodedFrame, decodeParams->qp);
 }
 
 // Decoder.
@@ -77,12 +84,19 @@ @implementation RTC_OBJC_TYPE (RTCVideoDecoderH264) {
   VTDecompressionSessionRef _decompressionSession;
   RTCVideoDecoderCallback _callback;
   OSStatus _error;
+  webrtc::H264BitstreamParser _h264BitstreamParser;
+  RTC_OBJC_TYPE(RTCVideoCodecInfo) *_codecInfo;
+
+#if !defined(DISABLE_H265)
+  webrtc::H265BitstreamParser _h265BitstreamParser;
+#endif
 }
 
-- (instancetype)init {
+- (instancetype)initWithCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)codecInfo {
   self = [super init];
   if (self) {
     _memoryPool = CMMemoryPoolCreate(nil);
+    _codecInfo = codecInfo;
   }
   return self;
 }
@@ -110,9 +124,20 @@ - (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
+#if !defined(DISABLE_H265)
+  rtc::ScopedCFTypeRef<CMVideoFormatDescriptionRef> inputFormat(nullptr);
+  if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) {
+    inputFormat = rtc::ScopedCF(webrtc::CreateVideoFormatDescription((uint8_t *)inputImage.buffer.bytes,
+                                                                     inputImage.buffer.length));
+  } else if (@available(iOS 11, *)) {
+    inputFormat = rtc::ScopedCF(webrtc::CreateH265VideoFormatDescription((uint8_t*)inputImage.buffer.bytes,
+                                                                         inputImage.buffer.length));
+  }
+#else
   rtc::ScopedCFTypeRef<CMVideoFormatDescriptionRef> inputFormat =
       rtc::ScopedCF(webrtc::CreateVideoFormatDescription((uint8_t *)inputImage.buffer.bytes,
                                                          inputImage.buffer.length));
+#endif
   if (inputFormat) {
     // Check if the video format has changed, and reinitialize decoder if
     // needed.
@@ -134,17 +159,54 @@ - (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   CMSampleBufferRef sampleBuffer = nullptr;
+#if !defined(DISABLE_H265)
+  if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) {
+    if (!webrtc::H264AnnexBBufferToCMSampleBuffer((uint8_t *)inputImage.buffer.bytes,
+                                                  inputImage.buffer.length,
+                                                  _videoFormat,
+                                                  &sampleBuffer,
+                                                  _memoryPool)) {
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+  } else if (@available(iOS 11, *)) {
+    if (!webrtc::H265AnnexBBufferToCMSampleBuffer((uint8_t*)inputImage.buffer.bytes,
+                                                  inputImage.buffer.length,
+                                                  _videoFormat,
+                                                  &sampleBuffer)) {
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+  } else {
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+#else
   if (!webrtc::H264AnnexBBufferToCMSampleBuffer((uint8_t *)inputImage.buffer.bytes,
                                                 inputImage.buffer.length,
                                                 _videoFormat,
                                                 &sampleBuffer,
                                                 _memoryPool)) {
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
+#endif
   RTC_DCHECK(sampleBuffer);
   VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression;
   std::unique_ptr<RTCFrameDecodeParams> frameDecodeParams;
-  frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp));
+
+  int qp = -1;
+#if !defined(DISABLE_H265)
+  if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) {
+    _h264BitstreamParser.ParseBitstream((uint8_t *)inputImage.buffer.bytes, inputImage.buffer.length);
+    _h264BitstreamParser.GetLastSliceQp(&qp);
+  } else {
+    _h265BitstreamParser.ParseBitstream((uint8_t *)inputImage.buffer.bytes, inputImage.buffer.length);
+    _h265BitstreamParser.GetLastSliceQp(&qp);
+  }
+#else
+  _h264BitstreamParser.ParseBitstream((uint8_t *)inputImage.buffer.bytes, inputImage.buffer.length);
+  _h264BitstreamParser.GetLastSliceQp(&qp);
+#endif
+
+  frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp, qp));
+
   OSStatus status = VTDecompressionSessionDecodeFrame(
       _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr);
 #if defined(WEBRTC_IOS)
@@ -154,7 +216,7 @@ - (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage
       [self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) {
     RTC_LOG(LS_INFO) << "Failed to decode frame with code: " << status
                      << " retrying decode after decompression session reset";
-    frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp));
+    frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp, qp));
     status = VTDecompressionSessionDecodeFrame(
         _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr);
   }

sdk/objc/components/video_codec/RTCVideoDecoderH265.h

@@ -10,10 +10,12 @@
 
 #import <Foundation/Foundation.h>
 
-#import "RTCMacros.h"
-#import "RTCVideoDecoder.h"
+#import "RTCVideoDecoderH264.h"
 
 RTC_OBJC_EXPORT
 API_AVAILABLE(ios(11.0))
-@interface RTCVideoDecoderH265 : NSObject <RTCVideoDecoder>
+@interface RTC_OBJC_TYPE (RTCVideoDecoderH265) : RTC_OBJC_TYPE(RTCVideoDecoderH264)
+
++ (bool)supported;
+
 @end