feat(gui): down scaled video files for fast thumbnail loading

Signed-off-by: Anton Kriese <[email protected]>

Author: akriese

GUI/src/main/kotlin/logic/FrameGrabber.kt

@@ -18,16 +18,20 @@ import java.awt.image.BufferedImage
  *
  * @param state [MutableState]<[AppState]> containing the global state. Needed to get the file paths to the videos.
  */
-class FrameGrabber(state: MutableState<AppState>) {
+class FrameGrabber(
+    referencePath: String,
+    currentPath: String,
+    diffPath: String? = null,
+    private val diffSequence: Array<AlignmentElement>,
+) {
     // create the grabbers
     private val videoReferenceGrabber: FFmpegFrameGrabber =
-        FFmpegFrameGrabber(state.value.videoReferencePath)
+        FFmpegFrameGrabber(referencePath)
     private val videoCurrentGrabber: FFmpegFrameGrabber =
-        FFmpegFrameGrabber(state.value.videoCurrentPath)
-    private val grabberDiff: FFmpegFrameGrabber = FFmpegFrameGrabber(state.value.outputPath)
+        FFmpegFrameGrabber(currentPath)
+    private val grabberDiff: FFmpegFrameGrabber? = if (diffPath == null) null else FFmpegFrameGrabber(diffPath)
 
     // create the sequences
-    private val diffSequence: Array<AlignmentElement> = state.value.sequenceObj
     private var videoReferenceFrames: MutableList<Int> = mutableListOf()
     private var videoCurrentFrames: MutableList<Int> = mutableListOf()
 
@@ -44,14 +48,14 @@ class FrameGrabber(state: MutableState<AppState>) {
         // start the grabbers
         videoReferenceGrabber.start()
         videoCurrentGrabber.start()
-        grabberDiff.start()
+        grabberDiff?.start()
 
         // generate the sequences for video 1 and video 2
         // diffSequence is already generated
         generateSequences()
 
-        width = grabberDiff.imageWidth
-        height = grabberDiff.imageHeight
+        width = grabberDiff?.imageWidth ?: videoReferenceGrabber.imageWidth
+        height = grabberDiff?.imageHeight ?: videoReferenceGrabber.imageHeight
 
         val coloredFrameGenerator = ColoredFrameGenerator(width, height)
         insertionBitmap =
@@ -98,6 +102,10 @@ class FrameGrabber(state: MutableState<AppState>) {
      * @return [ImageBitmap] containing the bitmap of the frame.
      */
     fun getDiffVideoFrame(index: Int): ImageBitmap {
+        if (grabberDiff == null) {
+            throw IllegalStateException("No difference video was provided.")
+        }
+
         grabberDiff.setVideoFrameNumber(index)
         return getBitmap(grabberDiff)
     }
@@ -203,9 +211,9 @@ class FrameGrabber(state: MutableState<AppState>) {
     fun close() {
         videoReferenceGrabber.stop()
         videoCurrentGrabber.stop()
-        grabberDiff.stop()
+        grabberDiff?.stop()
         videoReferenceGrabber.close()
         videoCurrentGrabber.close()
-        grabberDiff.close()
+        grabberDiff?.close()
     }
 }

GUI/src/main/kotlin/logic/ThumbnailVideoCreator.kt

@@ -0,0 +1,44 @@
+package logic
+
+import org.bytedeco.javacv.FFmpegFrameRecorder
+import wrappers.IterableFrameGrabber
+import wrappers.Resettable2DFrameConverter
+import java.awt.image.BufferedImage
+import java.io.File
+
+fun createThumbnailVideo(
+    inputPath: String,
+    downScaling: Float = 0.5f,
+): String {
+    assert(downScaling > 0.0f && downScaling <= 1.0f) { "Downscaling factor must be between 0 and 1" }
+    val outputPath = kotlin.io.path.createTempFile(prefix = "gui_thumbnail_video", suffix = ".mkv").toString()
+
+    val grabber = IterableFrameGrabber(File(inputPath))
+
+    val inWidth = grabber.imageWidth
+    val inHeight = grabber.imageHeight
+
+    val outWidth = (inWidth * downScaling).toInt()
+    val outHeight = (inHeight * downScaling).toInt()
+
+    val recorder = FFmpegFrameRecorder(outputPath, outWidth, outHeight)
+
+    val converter = Resettable2DFrameConverter()
+
+    recorder.frameRate = grabber.frameRate
+    recorder.videoCodec = grabber.videoCodec
+
+    recorder.start()
+
+    for (image in grabber) {
+        // do something with the frame
+        val scaledImage = BufferedImage(outWidth, outHeight, BufferedImage.TYPE_3BYTE_BGR)
+        scaledImage.createGraphics().drawImage(image.getScaledInstance(outWidth, outHeight, 0), 0, 0, null)
+        recorder.record(converter.convert(scaledImage))
+    }
+
+    recorder.close()
+    grabber.close()
+
+    return outputPath
+}

GUI/src/main/kotlin/models/AppState.kt

@@ -40,6 +40,8 @@ data class AppState(
     var hasUnsavedChanges: Boolean = false,
     var saveFramePath: String? = null,
     var saveInsertionsPath: String? = null,
+    var thumbnailVideoPathReference: String? = null,
+    var thumbnailVideoPathCurrent: String? = null,
     var maskPath: String? = null,
     var sequenceObj: Array<AlignmentElement> = arrayOf(),
     var gapOpenPenalty: Double = 0.2,

GUI/src/main/kotlin/ui/components/selectVideoScreen/ComputeDifferencesButton.kt

@@ -13,6 +13,7 @@ import androidx.compose.ui.Modifier
 import androidx.compose.ui.text.style.TextAlign
 import androidx.compose.ui.unit.dp
 import kotlinx.coroutines.*
+import logic.createThumbnailVideo
 import logic.differenceGeneratorWrapper.DifferenceGeneratorWrapper
 import logic.getVideoMetadata
 import models.AppState
@@ -45,12 +46,13 @@ fun RowScope.ComputeDifferencesButton(
         onClick = {
             try {
                 if (referenceIsOlderThanCurrent(state)) {
+                    createThumbnailVideos(state)
                     calculateVideoDifferences(scope, state, errorDialogText, showDialog)
                 } else {
                     showConfirmDialog.value = true
                 }
             } catch (e: Exception) {
-                errorDialogText.value = "An unexpected exception was thrown when checking" +
+                errorDialogText.value = "An unexpected exception was thrown when checking " +
                     "video creation timestamps:\n\n${e.message}"
             }
         },
@@ -78,6 +80,7 @@ fun RowScope.ComputeDifferencesButton(
         text = "The reference video is newer than the current video. Are you sure you want to continue?",
         showDialog = showConfirmDialog.value,
         onConfirm = {
+            createThumbnailVideos(state)
             calculateVideoDifferences(scope, state, errorDialogText, showDialog)
             showConfirmDialog.value = false
         },
@@ -87,6 +90,14 @@ fun RowScope.ComputeDifferencesButton(
     )
 }
 
+fun createThumbnailVideos(state: MutableState<AppState>) {
+    // create the thumbnail videos
+    val tempReference = createThumbnailVideo(state.value.videoReferencePath!!, 0.25f)
+    val tempCurrent = createThumbnailVideo(state.value.videoCurrentPath!!, 0.25f)
+
+    state.value = state.value.copy(thumbnailVideoPathReference = tempReference, thumbnailVideoPathCurrent = tempCurrent)
+}
+
 private fun calculateVideoDifferences(
     scope: CoroutineScope,
     state: MutableState<AppState>,

GUI/src/main/kotlin/ui/screens/DiffScreen.kt

@@ -46,8 +46,14 @@ fun DiffScreen(state: MutableState<AppState>) {
     // create the navigator, which implements the jumping logic
     val navigator = remember { FrameNavigation(state) }
     val showConfirmationDialog = remember { mutableStateOf(false) }
-    val frameGrabber = FrameGrabber(state)
-    val thumbnailGrabber = FrameGrabber(state)
+    val frameGrabber =
+        FrameGrabber(state.value.videoReferencePath!!, state.value.videoCurrentPath!!, state.value.outputPath!!, state.value.sequenceObj)
+    val thumbnailGrabber =
+        FrameGrabber(
+            state.value.thumbnailVideoPathReference!!,
+            state.value.thumbnailVideoPathCurrent!!,
+            diffSequence = state.value.sequenceObj,
+        )
 
     DisposableEffect(Unit) {
         onDispose {