Gui/feature/scaled tempvideos

GUI/src/main/kotlin/logic/FrameGrabber.kt

@@ -18,16 +18,20 @@ import java.awt.image.BufferedImage
  *
  * @param state [MutableState]<[AppState]> containing the global state. Needed to get the file paths to the videos.
  */
-class FrameGrabber(state: MutableState<AppState>) {
+class FrameGrabber(
+    referencePath: String,
+    currentPath: String,
+    diffPath: String? = null,
+    private val diffSequence: Array<AlignmentElement>,
+) {
     // create the grabbers
     private val videoReferenceGrabber: FFmpegFrameGrabber =
-        FFmpegFrameGrabber(state.value.videoReferencePath)
+        FFmpegFrameGrabber(referencePath)
     private val videoCurrentGrabber: FFmpegFrameGrabber =
-        FFmpegFrameGrabber(state.value.videoCurrentPath)
-    private val grabberDiff: FFmpegFrameGrabber = FFmpegFrameGrabber(state.value.outputPath)
+        FFmpegFrameGrabber(currentPath)
+    private val grabberDiff: FFmpegFrameGrabber? = if (diffPath == null) null else FFmpegFrameGrabber(diffPath)
 
     // create the sequences
-    private val diffSequence: Array<AlignmentElement> = state.value.sequenceObj
     private var videoReferenceFrames: MutableList<Int> = mutableListOf()
     private var videoCurrentFrames: MutableList<Int> = mutableListOf()
 
@@ -44,14 +48,14 @@ class FrameGrabber(state: MutableState<AppState>) {
         // start the grabbers
         videoReferenceGrabber.start()
         videoCurrentGrabber.start()
-        grabberDiff.start()
+        grabberDiff?.start()
 
         // generate the sequences for video 1 and video 2
         // diffSequence is already generated
         generateSequences()
 
-        width = grabberDiff.imageWidth
-        height = grabberDiff.imageHeight
+        width = grabberDiff?.imageWidth ?: videoReferenceGrabber.imageWidth
+        height = grabberDiff?.imageHeight ?: videoReferenceGrabber.imageHeight
 
         val coloredFrameGenerator = ColoredFrameGenerator(width, height)
         insertionBitmap =
@@ -98,6 +102,10 @@ class FrameGrabber(state: MutableState<AppState>) {
      * @return [ImageBitmap] containing the bitmap of the frame.
      */
     fun getDiffVideoFrame(index: Int): ImageBitmap {
+        if (grabberDiff == null) {
+            throw IllegalStateException("No difference video was provided.")
+        }
+
         grabberDiff.setVideoFrameNumber(index)
         return getBitmap(grabberDiff)
     }
@@ -203,9 +211,9 @@ class FrameGrabber(state: MutableState<AppState>) {
     fun close() {
         videoReferenceGrabber.stop()
         videoCurrentGrabber.stop()
-        grabberDiff.stop()
+        grabberDiff?.stop()
         videoReferenceGrabber.close()
         videoCurrentGrabber.close()
-        grabberDiff.close()
+        grabberDiff?.close()
     }
 }

GUI/src/main/kotlin/logic/ThumbnailVideoCreator.kt

@@ -0,0 +1,66 @@
+package logic
+
+import androidx.compose.runtime.MutableState
+import models.AppState
+import org.bytedeco.javacv.FFmpegFrameRecorder
+import wrappers.IterableFrameGrabber
+import wrappers.Resettable2DFrameConverter
+import java.awt.image.BufferedImage
+import java.io.File
+
+/**
+ * Creates a scaled video from the input video as a temp file.
+ *
+ * @param inputPath [String] containing the path to the input video.
+ * @param scale [Float] containing the scaling factor, must be greater than 0f.
+ * @return [String] containing the path to the scaled video.
+ */
+fun createScaledVideo(
+    inputPath: String,
+    scale: Float = 0.5f,
+): String {
+    assert(scale > 0.0f) { "Scaling factor must be positive!" }
+    val outputPath = kotlin.io.path.createTempFile(prefix = "gui_thumbnail_video", suffix = ".mkv").toString()
+
+    val grabber = IterableFrameGrabber(File(inputPath))
+
+    val inWidth = grabber.imageWidth
+    val inHeight = grabber.imageHeight
+
+    val outWidth = (inWidth * scale).toInt()
+    val outHeight = (inHeight * scale).toInt()
+
+    val recorder = FFmpegFrameRecorder(outputPath, outWidth, outHeight)
+    val converter = Resettable2DFrameConverter()
+
+    // copy some metadata over
+    recorder.frameRate = grabber.frameRate
+    recorder.videoCodec = grabber.videoCodec
+
+    recorder.start()
+
+    for (image in grabber) {
+        // scale down the image and record it
+        val scaledImage = BufferedImage(outWidth, outHeight, BufferedImage.TYPE_3BYTE_BGR)
+        scaledImage.createGraphics().drawImage(image.getScaledInstance(outWidth, outHeight, 0), 0, 0, null)
+        recorder.record(converter.convert(scaledImage))
+    }
+
+    recorder.close()
+    grabber.close()
+
+    return outputPath
+}
+
+/**
+ * Creates the thumbnail videos for the reference and current videos and updates their paths in the global state.
+ *
+ * @param state [MutableState]<[AppState]> containing the global state.
+ */
+fun createThumbnailVideos(state: MutableState<AppState>) {
+    // create the thumbnail videos
+    val tempReference = createScaledVideo(state.value.videoReferencePath!!, 0.25f)
+    val tempCurrent = createScaledVideo(state.value.videoCurrentPath!!, 0.25f)
+
+    state.value = state.value.copy(thumbnailVideoPathReference = tempReference, thumbnailVideoPathCurrent = tempCurrent)
+}

GUI/src/main/kotlin/models/AppState.kt

@@ -40,6 +40,8 @@ data class AppState(
     var hasUnsavedChanges: Boolean = false,
     var saveFramePath: String? = null,
     var saveInsertionsPath: String? = null,
+    var thumbnailVideoPathReference: String? = null,
+    var thumbnailVideoPathCurrent: String? = null,
     var maskPath: String? = null,
     var sequenceObj: Array<AlignmentElement> = arrayOf(),
     var gapOpenPenalty: Double = 0.2,

GUI/src/main/kotlin/ui/components/general/ProjectManager.kt

@@ -10,12 +10,14 @@ import androidx.compose.ui.text.font.FontWeight
 import androidx.compose.ui.unit.DpOffset
 import androidx.compose.ui.unit.dp
 import com.fasterxml.jackson.module.kotlin.readValue
+import logic.createThumbnailVideos
 import logic.getVideoMetadata
 import models.AppState
 import models.JsonMapper
 import org.bytedeco.javacv.FFmpegFrameGrabber
 import org.bytedeco.javacv.FFmpegFrameRecorder
 import org.bytedeco.javacv.Frame
+import java.io.File
 
 /**
  * Dropdown menu to open and save projects
@@ -131,6 +133,15 @@ fun handleOpenProject(
         state.value.outputPath = path
         // reset unsaved changes
         state.value.hasUnsavedChanges = false
+
+        // if the temp thumbnail videos exist, use them. Otherwise, create them again which might take a few seconds
+        if (!(
+                state.value.thumbnailVideoPathReference?.let { File(it).exists() } == true &&
+                    state.value.thumbnailVideoPathCurrent?.let { File(it).exists() } == true
+            )
+        ) {
+            createThumbnailVideos(state)
+        }
     } else {
         errorText.value = "The selected file does not contain a valid project."
     }

GUI/src/main/kotlin/ui/components/selectVideoScreen/ComputeDifferencesButton.kt

@@ -13,6 +13,7 @@ import androidx.compose.ui.Modifier
 import androidx.compose.ui.text.style.TextAlign
 import androidx.compose.ui.unit.dp
 import kotlinx.coroutines.*
+import logic.createThumbnailVideos
 import logic.differenceGeneratorWrapper.DifferenceGeneratorWrapper
 import logic.getVideoMetadata
 import models.AppState
@@ -45,12 +46,14 @@ fun RowScope.ComputeDifferencesButton(
         onClick = {
             try {
                 if (referenceIsOlderThanCurrent(state)) {
-                    calculateVideoDifferences(scope, state, errorDialogText, showDialog)
+                    scope.launch {
+                        runComputation(scope, state, errorDialogText, showDialog)
+                    }
                 } else {
                     showConfirmDialog.value = true
                 }
             } catch (e: Exception) {
-                errorDialogText.value = "An unexpected exception was thrown when checking" +
+                errorDialogText.value = "An unexpected exception was thrown when checking " +
                     "video creation timestamps:\n\n${e.message}"
             }
         },
@@ -78,7 +81,9 @@ fun RowScope.ComputeDifferencesButton(
         text = "The reference video is newer than the current video. Are you sure you want to continue?",
         showDialog = showConfirmDialog.value,
         onConfirm = {
-            calculateVideoDifferences(scope, state, errorDialogText, showDialog)
+            scope.launch {
+                runComputation(scope, state, errorDialogText, showDialog)
+            }
             showConfirmDialog.value = false
         },
         onCancel = {
@@ -87,50 +92,68 @@ fun RowScope.ComputeDifferencesButton(
     )
 }
 
-private fun calculateVideoDifferences(
+suspend fun runComputation(
     scope: CoroutineScope,
     state: MutableState<AppState>,
     errorDialogText: MutableState<String?>,
     isLoading: MutableState<Boolean>,
 ) {
-    scope.launch(Dispatchers.Default) {
-        isLoading.value = true
-        AlgorithmExecutionState.getInstance().reset()
-
-        // generate the differences
-        lateinit var generator: DifferenceGeneratorWrapper
-        try {
-            generator = DifferenceGeneratorWrapper(state)
-        } catch (e: DifferenceGeneratorException) {
-            errorDialogText.value = e.toString()
-            return@launch
-        } catch (e: Exception) {
-            errorDialogText.value = "An unexpected exception was thrown when creating" +
-                "the DifferenceGenerator instance:\n\n${e.message}"
-            return@launch
+    isLoading.value = true
+    val computeJob =
+        scope.launch(Dispatchers.Default) {
+            calculateVideoDifferences(state, errorDialogText)
         }
 
-        try {
-            generator.getDifferences(state.value.outputPath!!)
-        } catch (e: DifferenceGeneratorStoppedException) {
-            println("stopped by canceling...")
-            return@launch
-        } catch (e: Exception) {
-            errorDialogText.value = "An unexpected exception was thrown when running" +
-                "the difference computation:\n\n${e.message}"
-            return@launch
-        }
+    computeJob.invokeOnCompletion { isLoading.value = false }
 
-        // check for cancellation one last time before switching to the diff screen
-        if (!AlgorithmExecutionState.getInstance().isAlive()) {
-            return@launch
+    val videoScaleJob =
+        scope.launch(Dispatchers.Default) {
+            createThumbnailVideos(state)
         }
 
-        // set the sequence and screen
-        state.value = state.value.copy(sequenceObj = generator.getSequence(), screen = Screen.DiffScreen, hasUnsavedChanges = true)
-    }.invokeOnCompletion {
-        isLoading.value = false
+    // wait for both jobs to finish before transitioning to the diff screen
+    listOf(computeJob, videoScaleJob).joinAll()
+
+    state.value = state.value.copy(screen = Screen.DiffScreen, hasUnsavedChanges = true)
+}
+
+private fun calculateVideoDifferences(
+    state: MutableState<AppState>,
+    errorDialogText: MutableState<String?>,
+) {
+    AlgorithmExecutionState.getInstance().reset()
+
+    // generate the differences
+    lateinit var generator: DifferenceGeneratorWrapper
+    try {
+        generator = DifferenceGeneratorWrapper(state)
+    } catch (e: DifferenceGeneratorException) {
+        errorDialogText.value = e.toString()
+        return
+    } catch (e: Exception) {
+        errorDialogText.value = "An unexpected exception was thrown when creating" +
+            "the DifferenceGenerator instance:\n\n${e.message}"
+        return
+    }
+
+    try {
+        generator.getDifferences(state.value.outputPath!!)
+    } catch (e: DifferenceGeneratorStoppedException) {
+        println("stopped by canceling...")
+        return
+    } catch (e: Exception) {
+        errorDialogText.value = "An unexpected exception was thrown when running" +
+            "the difference computation:\n\n${e.message}"
+        return
+    }
+
+    // check for cancellation one last time before switching to the diff screen
+    if (!AlgorithmExecutionState.getInstance().isAlive()) {
+        return
     }
+
+    // set the sequence
+    state.value = state.value.copy(sequenceObj = generator.getSequence())
 }
 
 fun getVideoCreationDate(videoPath: String): Long {

GUI/src/main/kotlin/ui/screens/DiffScreen.kt

@@ -46,8 +46,14 @@ fun DiffScreen(state: MutableState<AppState>) {
     // create the navigator, which implements the jumping logic
     val navigator = remember { FrameNavigation(state) }
     val showConfirmationDialog = remember { mutableStateOf(false) }
-    val frameGrabber = FrameGrabber(state)
-    val thumbnailGrabber = FrameGrabber(state)
+    val frameGrabber =
+        FrameGrabber(state.value.videoReferencePath!!, state.value.videoCurrentPath!!, state.value.outputPath!!, state.value.sequenceObj)
+    val thumbnailGrabber =
+        FrameGrabber(
+            state.value.thumbnailVideoPathReference!!,
+            state.value.thumbnailVideoPathCurrent!!,
+            diffSequence = state.value.sequenceObj,
+        )
 
     DisposableEffect(Unit) {
         onDispose {