|
| 1 | +// This file is part of OpenCV project. |
| 2 | +// It is subject to the license terms in the LICENSE file found in the top-level directory |
| 3 | +// of this distribution and at http://opencv.org/license.html |
| 4 | + |
| 5 | +#include "precomp.hpp" |
| 6 | + |
| 7 | +#include <stdio.h> |
| 8 | +#include <string.h> |
| 9 | +#include <fstream> |
| 10 | +#include <iostream> |
| 11 | +#include <unistd.h> |
| 12 | +#include <sys/stat.h> |
| 13 | +#include <sys/types.h> |
| 14 | +#include <fcntl.h> |
| 15 | +#include <android/log.h> |
| 16 | + |
| 17 | +#include "media/NdkMediaCodec.h" |
| 18 | +#include "media/NdkMediaExtractor.h" |
| 19 | + |
| 20 | +#define INPUT_TIMEOUT_MS 2000 |
| 21 | + |
| 22 | +#define COLOR_FormatYUV420Planar 19 |
| 23 | +#define COLOR_FormatYUV420SemiPlanar 21 |
| 24 | + |
| 25 | +using namespace cv; |
| 26 | + |
| 27 | +#define TAG "NativeCodec" |
| 28 | +#define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__) |
| 29 | +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) |
| 30 | + |
| 31 | + |
| 32 | +static inline void deleter_AMediaExtractor(AMediaExtractor *extractor) { |
| 33 | + AMediaExtractor_delete(extractor); |
| 34 | +} |
| 35 | + |
| 36 | +static inline void deleter_AMediaCodec(AMediaCodec *codec) { |
| 37 | + AMediaCodec_stop(codec); |
| 38 | + AMediaCodec_delete(codec); |
| 39 | +} |
| 40 | + |
| 41 | +static inline void deleter_AMediaFormat(AMediaFormat *format) { |
| 42 | + AMediaFormat_delete(format); |
| 43 | +} |
| 44 | + |
| 45 | +class AndroidMediaNdkCapture : public IVideoCapture |
| 46 | +{ |
| 47 | + |
| 48 | +public: |
| 49 | + AndroidMediaNdkCapture(): |
| 50 | + sawInputEOS(false), sawOutputEOS(false), |
| 51 | + frameWidth(0), frameHeight(0), colorFormat(0) {} |
| 52 | + std::shared_ptr<AMediaExtractor> mediaExtractor; |
| 53 | + std::shared_ptr<AMediaCodec> mediaCodec; |
| 54 | + bool sawInputEOS; |
| 55 | + bool sawOutputEOS; |
| 56 | + int32_t frameWidth; |
| 57 | + int32_t frameHeight; |
| 58 | + int32_t colorFormat; |
| 59 | + std::vector<uint8_t> buffer; |
| 60 | + |
| 61 | + ~AndroidMediaNdkCapture() { cleanUp(); } |
| 62 | + |
| 63 | + bool decodeFrame() { |
| 64 | + while (!sawInputEOS || !sawOutputEOS) { |
| 65 | + if (!sawInputEOS) { |
| 66 | + auto bufferIndex = AMediaCodec_dequeueInputBuffer(mediaCodec.get(), INPUT_TIMEOUT_MS); |
| 67 | + LOGV("input buffer %zd", bufferIndex); |
| 68 | + if (bufferIndex >= 0) { |
| 69 | + size_t bufferSize; |
| 70 | + auto inputBuffer = AMediaCodec_getInputBuffer(mediaCodec.get(), bufferIndex, &bufferSize); |
| 71 | + auto sampleSize = AMediaExtractor_readSampleData(mediaExtractor.get(), inputBuffer, bufferSize); |
| 72 | + if (sampleSize < 0) { |
| 73 | + sampleSize = 0; |
| 74 | + sawInputEOS = true; |
| 75 | + LOGV("EOS"); |
| 76 | + } |
| 77 | + auto presentationTimeUs = AMediaExtractor_getSampleTime(mediaExtractor.get()); |
| 78 | + |
| 79 | + AMediaCodec_queueInputBuffer(mediaCodec.get(), bufferIndex, 0, sampleSize, |
| 80 | + presentationTimeUs, sawInputEOS ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0); |
| 81 | + AMediaExtractor_advance(mediaExtractor.get()); |
| 82 | + } |
| 83 | + } |
| 84 | + |
| 85 | + if (!sawOutputEOS) { |
| 86 | + AMediaCodecBufferInfo info; |
| 87 | + auto bufferIndex = AMediaCodec_dequeueOutputBuffer(mediaCodec.get(), &info, 0); |
| 88 | + if (bufferIndex >= 0) { |
| 89 | + size_t bufferSize = 0; |
| 90 | + auto mediaFormat = std::shared_ptr<AMediaFormat>( AMediaCodec_getOutputFormat(mediaCodec. get()), deleter_AMediaFormat); |
| 91 | + AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_WIDTH, &frameWidth); |
| 92 | + AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_HEIGHT, &frameHeight); |
| 93 | + AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat); |
| 94 | + uint8_t* codecBuffer = AMediaCodec_getOutputBuffer(mediaCodec.get(), bufferIndex, &bufferSize); |
| 95 | + buffer = std::vector<uint8_t>(codecBuffer + info.offset, codecBuffer + bufferSize); |
| 96 | + LOGV("colorFormat: %d", colorFormat); |
| 97 | + LOGV("buffer size: %zu", bufferSize); |
| 98 | + LOGV("width (frame): %d", frameWidth); |
| 99 | + LOGV("height (frame): %d", frameHeight); |
| 100 | + if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) { |
| 101 | + LOGV("output EOS"); |
| 102 | + sawOutputEOS = true; |
| 103 | + } |
| 104 | + AMediaCodec_releaseOutputBuffer(mediaCodec.get(), bufferIndex, info.size != 0); |
| 105 | + return true; |
| 106 | + } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) { |
| 107 | + LOGV("output buffers changed"); |
| 108 | + } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) { |
| 109 | + auto format = AMediaCodec_getOutputFormat(mediaCodec.get()); |
| 110 | + LOGV("format changed to: %s", AMediaFormat_toString(format)); |
| 111 | + AMediaFormat_delete(format); |
| 112 | + } else if (bufferIndex == AMEDIACODEC_INFO_TRY_AGAIN_LATER) { |
| 113 | + LOGV("no output buffer right now"); |
| 114 | + } else { |
| 115 | + LOGV("unexpected info code: %zd", bufferIndex); |
| 116 | + } |
| 117 | + } |
| 118 | + } |
| 119 | + return false; |
| 120 | + } |
| 121 | + |
| 122 | + bool isOpened() const CV_OVERRIDE { return mediaCodec.get() != nullptr; } |
| 123 | + |
| 124 | + int getCaptureDomain() CV_OVERRIDE { return CAP_ANDROID; } |
| 125 | + |
| 126 | + bool grabFrame() CV_OVERRIDE |
| 127 | + { |
| 128 | + // clear the previous frame |
| 129 | + buffer.clear(); |
| 130 | + return decodeFrame(); |
| 131 | + } |
| 132 | + |
| 133 | + bool retrieveFrame(int, OutputArray out) CV_OVERRIDE |
| 134 | + { |
| 135 | + if (buffer.empty()) { |
| 136 | + return false; |
| 137 | + } |
| 138 | + Mat yuv(frameHeight + frameHeight/2, frameWidth, CV_8UC1, buffer.data()); |
| 139 | + if (colorFormat == COLOR_FormatYUV420Planar) { |
| 140 | + cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_YV12); |
| 141 | + } else if (colorFormat == COLOR_FormatYUV420SemiPlanar) { |
| 142 | + cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_NV21); |
| 143 | + } else { |
| 144 | + LOGE("Unsupported video format: %d", colorFormat); |
| 145 | + return false; |
| 146 | + } |
| 147 | + return true; |
| 148 | + } |
| 149 | + |
| 150 | + double getProperty(int property_id) const CV_OVERRIDE |
| 151 | + { |
| 152 | + switch (property_id) |
| 153 | + { |
| 154 | + case CV_CAP_PROP_FRAME_WIDTH: return frameWidth; |
| 155 | + case CV_CAP_PROP_FRAME_HEIGHT: return frameHeight; |
| 156 | + } |
| 157 | + return 0; |
| 158 | + } |
| 159 | + |
| 160 | + bool setProperty(int /* property_id */, double /* value */) CV_OVERRIDE |
| 161 | + { |
| 162 | + return false; |
| 163 | + } |
| 164 | + |
| 165 | + bool initCapture(const char * filename) |
| 166 | + { |
| 167 | + struct stat statBuffer; |
| 168 | + if (stat(filename, &statBuffer) != 0) { |
| 169 | + LOGE("failed to stat file: %s (%s)", filename, strerror(errno)); |
| 170 | + return false; |
| 171 | + } |
| 172 | + |
| 173 | + int fd = open(filename, O_RDONLY); |
| 174 | + |
| 175 | + if (fd < 0) { |
| 176 | + LOGE("failed to open file: %s %d (%s)", filename, fd, strerror(errno)); |
| 177 | + return false; |
| 178 | + } |
| 179 | + |
| 180 | + mediaExtractor = std::shared_ptr<AMediaExtractor>(AMediaExtractor_new(), deleter_AMediaExtractor); |
| 181 | + if (!mediaExtractor) { |
| 182 | + return false; |
| 183 | + } |
| 184 | + media_status_t err = AMediaExtractor_setDataSourceFd(mediaExtractor.get(), fd, 0, statBuffer.st_size); |
| 185 | + close(fd); |
| 186 | + if (err != AMEDIA_OK) { |
| 187 | + LOGV("setDataSource error: %d", err); |
| 188 | + return false; |
| 189 | + } |
| 190 | + |
| 191 | + int numtracks = AMediaExtractor_getTrackCount(mediaExtractor.get()); |
| 192 | + |
| 193 | + LOGV("input has %d tracks", numtracks); |
| 194 | + for (int i = 0; i < numtracks; i++) { |
| 195 | + auto format = std::shared_ptr<AMediaFormat>( AMediaExtractor_getTrackFormat(mediaExtractor. get(), i), deleter_AMediaFormat); |
| 196 | + if (!format) { |
| 197 | + continue; |
| 198 | + } |
| 199 | + const char *s = AMediaFormat_toString(format.get()); |
| 200 | + LOGV("track %d format: %s", i, s); |
| 201 | + const char *mime; |
| 202 | + if (!AMediaFormat_getString(format.get(), AMEDIAFORMAT_KEY_MIME, &mime)) { |
| 203 | + LOGV("no mime type"); |
| 204 | + } else if (!strncmp(mime, "video/", 6)) { |
| 205 | + int32_t trackWidth, trackHeight; |
| 206 | + AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_WIDTH, &trackWidth); |
| 207 | + AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_HEIGHT, &trackHeight); |
| 208 | + LOGV("width (track): %d", trackWidth); |
| 209 | + LOGV("height (track): %d", trackHeight); |
| 210 | + if (AMediaExtractor_selectTrack(mediaExtractor.get(), i) != AMEDIA_OK) { |
| 211 | + continue; |
| 212 | + } |
| 213 | + mediaCodec = std::shared_ptr<AMediaCodec>(AMediaCodec_createDecoderByType(mime), deleter_AMediaCodec); |
| 214 | + if (!mediaCodec) { |
| 215 | + continue; |
| 216 | + } |
| 217 | + if (AMediaCodec_configure(mediaCodec.get(), format.get(), NULL, NULL, 0) != AMEDIA_OK) { |
| 218 | + continue; |
| 219 | + } |
| 220 | + sawInputEOS = false; |
| 221 | + sawOutputEOS = false; |
| 222 | + if (AMediaCodec_start(mediaCodec.get()) != AMEDIA_OK) { |
| 223 | + continue; |
| 224 | + } |
| 225 | + return true; |
| 226 | + } |
| 227 | + } |
| 228 | + |
| 229 | + return false; |
| 230 | + } |
| 231 | + |
| 232 | + void cleanUp() { |
| 233 | + sawInputEOS = true; |
| 234 | + sawOutputEOS = true; |
| 235 | + frameWidth = 0; |
| 236 | + frameHeight = 0; |
| 237 | + colorFormat = 0; |
| 238 | + } |
| 239 | +}; |
| 240 | + |
| 241 | +/****************** Implementation of interface functions ********************/ |
| 242 | + |
| 243 | +Ptr<IVideoCapture> cv::createAndroidCapture_file(const std::string &filename) { |
| 244 | + Ptr<AndroidMediaNdkCapture> res = makePtr<AndroidMediaNdkCapture>(); |
| 245 | + if (res && res->initCapture(filename.c_str())) |
| 246 | + return res; |
| 247 | + return Ptr<IVideoCapture>(); |
| 248 | +} |
0 commit comments