Skip to content

Commit a4a7166

Browse files
KoiFreshKai Mayer
KoiFresh
and
Kai Mayer
authored
Add implementation to MediaStreamTrack.captureFrame() for linux (#1390)
* feat: add flutter_frame_capturer * feat: use svpng to store png file feat: use svpng to store png file fix: use lodepng instead of libpng feat: use svpng to store png file * fix: add flutter_frame_capturer.cc in add_library * feat: add sample and use fopne_s on windows * fix: add _CRT_SECURE_NO_WARNINGS * fix: preprocessor ifndef FLUTTER_WEBRTC_RTC_FRAME_CAPTURER_HXX --------- Co-authored-by: Kai Mayer <[email protected]>
1 parent b489458 commit a4a7166

File tree

9 files changed

+346
-27
lines changed

9 files changed

+346
-27
lines changed

‎common/cpp/include/flutter_frame_capturer.h

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#ifndef FLUTTER_WEBRTC_RTC_FRAME_CAPTURER_HXX
2+
#define FLUTTER_WEBRTC_RTC_FRAME_CAPTURER_HXX
3+
4+
#include "flutter_common.h"
5+
#include "flutter_webrtc_base.h"
6+
7+
#include "rtc_video_frame.h"
8+
#include "rtc_video_renderer.h"
9+
10+
#include <mutex>
11+
12+
namespace flutter_webrtc_plugin {
13+
14+
using namespace libwebrtc;
15+
16+
class FlutterFrameCapturer
17+
: public RTCVideoRenderer<scoped_refptr<RTCVideoFrame>> {
18+
public:
19+
FlutterFrameCapturer(RTCVideoTrack* track, std::string path);
20+
21+
virtual void OnFrame(scoped_refptr<RTCVideoFrame> frame) override;
22+
23+
void CaptureFrame(std::unique_ptr<MethodResultProxy> result);
24+
25+
private:
26+
RTCVideoTrack* track_;
27+
std::string path_;
28+
std::mutex mutex_;
29+
scoped_refptr<RTCVideoFrame> frame_;
30+
31+
bool SaveFrame();
32+
};
33+
34+
} // namespace flutter_webrtc_plugin
35+
36+
#endif // !FLUTTER_WEBRTC_RTC_FRAME_CAPTURER_HXX

‎common/cpp/src/flutter_frame_capturer.cc

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#ifdef _MSC_VER
2+
#define _CRT_SECURE_NO_WARNINGS
3+
#endif
4+
5+
#include "flutter_frame_capturer.h"
6+
#include <stdio.h>
7+
#include <stdlib.h>
8+
#include "svpng.hpp"
9+
10+
namespace flutter_webrtc_plugin {
11+
12+
FlutterFrameCapturer::FlutterFrameCapturer(RTCVideoTrack* track,
13+
std::string path) {
14+
track_ = track;
15+
path_ = path;
16+
}
17+
18+
void FlutterFrameCapturer::OnFrame(scoped_refptr<RTCVideoFrame> frame) {
19+
if (frame_ != nullptr) {
20+
return;
21+
}
22+
23+
frame_ = frame.get()->Copy();
24+
mutex_.unlock();
25+
}
26+
27+
void FlutterFrameCapturer::CaptureFrame(
28+
std::unique_ptr<MethodResultProxy> result) {
29+
mutex_.lock();
30+
track_->AddRenderer(this);
31+
// Here the OnFrame method has to unlock the mutex
32+
mutex_.lock();
33+
track_->RemoveRenderer(this);
34+
35+
bool success = SaveFrame();
36+
mutex_.unlock();
37+
38+
std::shared_ptr<MethodResultProxy> result_ptr(result.release());
39+
if (success) {
40+
result_ptr->Success();
41+
} else {
42+
result_ptr->Error("1", "Cannot save the frame as .png file");
43+
}
44+
}
45+
46+
bool FlutterFrameCapturer::SaveFrame() {
47+
if (frame_ == nullptr) {
48+
return false;
49+
}
50+
51+
int width = frame_.get()->width();
52+
int height = frame_.get()->height();
53+
int bytes_per_pixel = 4;
54+
uint8_t* pixels = new uint8_t[width * height * bytes_per_pixel];
55+
56+
frame_.get()->ConvertToARGB(RTCVideoFrame::Type::kABGR, pixels,
57+
/* unused */ -1, width, height);
58+
59+
FILE* file = fopen(path_.c_str(), "wb");
60+
if (!file) {
61+
return false;
62+
}
63+
64+
svpng(file, width, height, pixels, 1);
65+
fclose(file);
66+
return true;
67+
}
68+
69+
} // namespace flutter_webrtc_plugin

‎common/cpp/src/flutter_peerconnection.cc

+35-27
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include "base/scoped_ref_ptr.h"
44
#include "flutter_data_channel.h"
5+
#include "flutter_frame_capturer.h"
56
#include "rtc_dtmf_sender.h"
67
#include "rtc_rtp_parameters.h"
78

@@ -107,21 +108,26 @@ EncodableMap rtpParametersToMap(
107108
}
108109
info[EncodableValue("codecs")] = EncodableValue(codecs_info);
109110

110-
switch(rtpParameters->GetDegradationPreference()) {
111+
switch (rtpParameters->GetDegradationPreference()) {
111112
case libwebrtc::RTCDegradationPreference::MAINTAIN_FRAMERATE:
112-
info[EncodableValue("degradationPreference")] = EncodableValue("maintain-framerate");
113+
info[EncodableValue("degradationPreference")] =
114+
EncodableValue("maintain-framerate");
113115
break;
114116
case libwebrtc::RTCDegradationPreference::MAINTAIN_RESOLUTION:
115-
info[EncodableValue("degradationPreference")] = EncodableValue("maintain-resolution");
117+
info[EncodableValue("degradationPreference")] =
118+
EncodableValue("maintain-resolution");
116119
break;
117120
case libwebrtc::RTCDegradationPreference::BALANCED:
118-
info[EncodableValue("degradationPreference")] = EncodableValue("balanced");
121+
info[EncodableValue("degradationPreference")] =
122+
EncodableValue("balanced");
119123
break;
120124
case libwebrtc::RTCDegradationPreference::DISABLED:
121-
info[EncodableValue("degradationPreference")] = EncodableValue("disabled");
125+
info[EncodableValue("degradationPreference")] =
126+
EncodableValue("disabled");
122127
break;
123128
default:
124-
info[EncodableValue("degradationPreference")] = EncodableValue("balanced");
129+
info[EncodableValue("degradationPreference")] =
130+
EncodableValue("balanced");
125131
break;
126132
}
127133

@@ -205,7 +211,8 @@ EncodableMap rtpReceiverToMap(
205211

206212
EncodableMap transceiverToMap(scoped_refptr<RTCRtpTransceiver> transceiver) {
207213
EncodableMap info;
208-
info[EncodableValue("transceiverId")] = EncodableValue(transceiver->transceiver_id().std_string());
214+
info[EncodableValue("transceiverId")] =
215+
EncodableValue(transceiver->transceiver_id().std_string());
209216
info[EncodableValue("mid")] = EncodableValue(transceiver->mid().std_string());
210217
info[EncodableValue("direction")] =
211218
EncodableValue(transceiverDirectionString(transceiver->direction()));
@@ -272,7 +279,6 @@ void FlutterPeerConnection::RTCPeerConnectionClose(
272279
RTCPeerConnection* pc,
273280
const std::string& uuid,
274281
std::unique_ptr<MethodResultProxy> result) {
275-
276282
auto it2 = base_->peerconnections_.find(uuid);
277283
if (it2 != base_->peerconnections_.end()) {
278284
it2->second->Close();
@@ -282,7 +288,7 @@ void FlutterPeerConnection::RTCPeerConnectionClose(
282288
auto it = base_->peerconnection_observers_.find(uuid);
283289
if (it != base_->peerconnection_observers_.end())
284290
base_->peerconnection_observers_.erase(it);
285-
291+
286292
result->Success();
287293
}
288294

@@ -643,18 +649,23 @@ scoped_refptr<RTCRtpParameters> FlutterPeerConnection::updateRtpParameters(
643649
encoding++;
644650
}
645651
}
646-
647-
EncodableValue value = findEncodableValue(newParameters, "degradationPreference");
648-
if(!value.IsNull()) {
652+
653+
EncodableValue value =
654+
findEncodableValue(newParameters, "degradationPreference");
655+
if (!value.IsNull()) {
649656
const std::string degradationPreference = GetValue<std::string>(value);
650-
if( degradationPreference == "maintain-framerate") {
651-
parameters->SetDegradationPreference(libwebrtc::RTCDegradationPreference::MAINTAIN_FRAMERATE);
652-
} else if (degradationPreference == "maintain-resolution") {
653-
parameters->SetDegradationPreference(libwebrtc::RTCDegradationPreference::MAINTAIN_RESOLUTION);
654-
} else if(degradationPreference == "balanced") {
655-
parameters->SetDegradationPreference(libwebrtc::RTCDegradationPreference::BALANCED);
656-
} else if(degradationPreference == "disabled") {
657-
parameters->SetDegradationPreference(libwebrtc::RTCDegradationPreference::DISABLED);
657+
if (degradationPreference == "maintain-framerate") {
658+
parameters->SetDegradationPreference(
659+
libwebrtc::RTCDegradationPreference::MAINTAIN_FRAMERATE);
660+
} else if (degradationPreference == "maintain-resolution") {
661+
parameters->SetDegradationPreference(
662+
libwebrtc::RTCDegradationPreference::MAINTAIN_RESOLUTION);
663+
} else if (degradationPreference == "balanced") {
664+
parameters->SetDegradationPreference(
665+
libwebrtc::RTCDegradationPreference::BALANCED);
666+
} else if (degradationPreference == "disabled") {
667+
parameters->SetDegradationPreference(
668+
libwebrtc::RTCDegradationPreference::DISABLED);
658669
}
659670
}
660671

@@ -711,8 +722,8 @@ void FlutterPeerConnection::RtpTransceiverGetCurrentDirection(
711722
return;
712723
}
713724
EncodableMap map;
714-
map[EncodableValue("result")] =
715-
EncodableValue(transceiverDirectionString(transceiver->current_direction()));
725+
map[EncodableValue("result")] = EncodableValue(
726+
transceiverDirectionString(transceiver->current_direction()));
716727
result_ptr->Success(EncodableValue(map));
717728
}
718729

@@ -731,11 +742,8 @@ void FlutterPeerConnection::CaptureFrame(
731742
RTCVideoTrack* track,
732743
std::string path,
733744
std::unique_ptr<MethodResultProxy> result) {
734-
std::shared_ptr<MethodResultProxy> result_ptr(result.release());
735-
736-
// TODO pc->CaptureFrame();
737-
738-
result_ptr->Success();
745+
FlutterFrameCapturer capturer(track, path);
746+
capturer.CaptureFrame(std::move(result));
739747
}
740748

741749
scoped_refptr<RTCRtpTransceiver> FlutterPeerConnection::getRtpTransceiverById(

‎example/lib/main.dart

+9
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import 'package:flutter/foundation.dart'
55
import 'package:flutter/material.dart';
66
import 'package:flutter_background/flutter_background.dart';
77
import 'package:flutter_webrtc/flutter_webrtc.dart';
8+
import 'package:flutter_webrtc_example/src/capture_frame_sample.dart';
89

910
import 'src/device_enumeration_sample.dart';
1011
import 'src/get_display_media_sample.dart';
@@ -126,6 +127,14 @@ class _MyAppState extends State<MyApp> {
126127
builder: (BuildContext context) =>
127128
DataChannelLoopBackSample()));
128129
}),
130+
RouteItem(
131+
title: 'Capture Frame',
132+
push: (BuildContext context) {
133+
Navigator.push(
134+
context,
135+
MaterialPageRoute(
136+
builder: (BuildContext context) => CaptureFrameSample()));
137+
}),
129138
];
130139
}
131140
}

‎example/lib/src/capture_frame_sample.dart

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import 'dart:typed_data';
2+
3+
import 'package:flutter/material.dart';
4+
import 'package:flutter_webrtc/flutter_webrtc.dart';
5+
6+
class CaptureFrameSample extends StatefulWidget {
7+
@override
8+
State<StatefulWidget> createState() => _CaptureFrameSample();
9+
}
10+
11+
class _CaptureFrameSample extends State<CaptureFrameSample> {
12+
Uint8List? _data;
13+
14+
void _captureFrame() async {
15+
final stream = await navigator.mediaDevices.getUserMedia({
16+
'audio': false,
17+
'video': true,
18+
});
19+
20+
final track = stream.getVideoTracks().first;
21+
final buffer = await track.captureFrame();
22+
23+
stream.getTracks().forEach((track) => track.stop());
24+
25+
setState(() {
26+
_data = buffer.asUint8List();
27+
});
28+
}
29+
30+
@override
31+
Widget build(BuildContext context) {
32+
return Scaffold(
33+
appBar: AppBar(
34+
title: const Text('Capture Frame'),
35+
),
36+
floatingActionButton: FloatingActionButton(
37+
onPressed: _captureFrame,
38+
child: Icon(Icons.camera_alt_outlined),
39+
),
40+
body: Builder(builder: (context) {
41+
final data = _data;
42+
43+
if (data == null) {
44+
return Container();
45+
}
46+
return Center(
47+
child: Image.memory(
48+
data,
49+
fit: BoxFit.contain,
50+
width: double.infinity,
51+
height: double.infinity,
52+
),
53+
);
54+
}),
55+
);
56+
}
57+
}

‎linux/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ add_library(${PLUGIN_NAME} SHARED
1414
"../common/cpp/src/flutter_frame_cryptor.cc"
1515
"../common/cpp/src/flutter_media_stream.cc"
1616
"../common/cpp/src/flutter_peerconnection.cc"
17+
"../common/cpp/src/flutter_frame_capturer.cc"
1718
"../common/cpp/src/flutter_video_renderer.cc"
1819
"../common/cpp/src/flutter_screen_capture.cc"
1920
"../common/cpp/src/flutter_webrtc.cc"
@@ -31,6 +32,7 @@ include_directories(
3132
"${CMAKE_CURRENT_SOURCE_DIR}/../common/cpp/include"
3233
"${CMAKE_CURRENT_SOURCE_DIR}/../third_party/uuidxx"
3334
"${CMAKE_CURRENT_SOURCE_DIR}/../third_party/libwebrtc/include"
35+
"${CMAKE_CURRENT_SOURCE_DIR}/../third_party/svpng"
3436
)
3537

3638
apply_standard_settings(${PLUGIN_NAME})

‎third_party/svpng/LICENSE

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
Copyright (C) 2017 Milo Yip. All rights reserved.
2+
3+
Redistribution and use in source and binary forms, with or without
4+
modification, are permitted provided that the following conditions are met:
5+
6+
* Redistributions of source code must retain the above copyright notice, this
7+
list of conditions and the following disclaimer.
8+
9+
* Redistributions in binary form must reproduce the above copyright notice,
10+
this list of conditions and the following disclaimer in the documentation
11+
and/or other materials provided with the distribution.
12+
13+
* Neither the name of pngout nor the names of its
14+
contributors may be used to endorse or promote products derived from
15+
this software without specific prior written permission.
16+
17+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
21+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
24+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
25+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

0 commit comments

Comments
 (0)