Optional per-dataset metadata file passed to opt-in pipelines

client/dive-common/apispec.ts

@@ -54,6 +54,20 @@ interface PipeMetadata {
   outputType?: string;
   diveParams?: DiveParam[];
   requiresCalibration?: boolean;
+  /**
+   * KWIVER config key (e.g. "stabilizer:flight_log") that the dataset's optional
+   * metadata file should be bound to at run time. Parsed from a pipe header
+   * `# Metadata File: <block>:<key>`. When unset, the pipe does not consume a
+   * metadata file and none is injected.
+   */
+  metadataFileKey?: string;
+  /**
+   * KWIVER config key templates (e.g. "stabilizer:image_list{cam}") bound to the
+   * run's per-camera input image lists — one single-file, line-separated list per
+   * camera. A `{cam}` placeholder is expanded per camera (1-based); a key without
+   * it gets the first camera's list. Parsed from a `# Image List Keys:` header.
+   */
+  imageListKeys?: string[];
 }
 
 interface PipelineRuntimeParams {
@@ -154,6 +168,7 @@ export interface MultiCamImportFolderArgs {
     glob?: string;
   }>; // path/track file per camera
   calibrationFile?: string; // NPZ calibation matrix file
+  metadataFile?: string; // Optional per-dataset metadata file (e.g. sea-lion flight log)
   type: 'image-sequence' | 'video' | 'large-image';
 }
 
@@ -165,6 +180,7 @@ export interface MultiCamImportKeywordArgs {
     trackFile: string;
   }>; // glob pattern for base folder
   calibrationFile?: string; // NPZ calibation matrix file
+  metadataFile?: string; // Optional per-dataset metadata file (e.g. sea-lion flight log)
   type: 'image-sequence'; // Always image-sequence type for glob matching
 }
 
@@ -222,6 +238,8 @@ interface DatasetMeta extends DatasetMetaMutable {
   multiCamMedia: Readonly<MultiCamMedia | null>;
   /** Stereo calibration / camera file currently associated with the dataset (desktop). */
   calibration?: Readonly<string | null>;
+  /** Optional metadata file associated with the dataset, passed to opt-in pipelines. */
+  metadataFile?: Readonly<string | null>;
 }
 
 interface CameraCalibration {
@@ -297,7 +315,7 @@ interface Api {
   saveAttributeTrackFilters(datasetId: string,
     args: SaveAttributeTrackFilterArgs): Promise<unknown>;
   // Non-Endpoint shared functions
-  openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip' | 'transform', directory?: boolean):
+  openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip' | 'transform' | 'metadata', directory?: boolean):
     Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>;
   /** Desktop: immediate child directory names under a parent folder (multicam subfolder import). */
   listImmediateSubfolders?(parentPath: string): Promise<string[]>;

client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue

@@ -14,6 +14,7 @@ import ImportMultiCamMultiFolder from './ImportMultiCamMultiFolder.vue';
 import ImportMultiCamKeyword from './ImportMultiCamKeyword.vue';
 import ImportMultiCamFinalizeStep from './ImportMultiCamFinalizeStep.vue';
 import ImportMultiCamCalibration from './ImportMultiCamCalibration.vue';
+import ImportMultiCamMetadata from './ImportMultiCamMetadata.vue';
 
 export default defineComponent({
   name: 'ImportMultiCamDialog',
@@ -24,6 +25,7 @@ export default defineComponent({
     ImportMultiCamKeyword,
     ImportMultiCamFinalizeStep,
     ImportMultiCamCalibration,
+    ImportMultiCamMetadata,
   },
   props: {
     stereo: {
@@ -117,6 +119,11 @@ export default defineComponent({
         :ctx="ctx"
       />
 
+      <ImportMultiCamMetadata
+        v-if="importType"
+        :ctx="ctx"
+      />
+
       <div>
         <v-alert
           v-if="errorMessage"

client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMetadata.vue

@@ -0,0 +1,65 @@
+<!--
+  Optional per-dataset metadata file picker (e.g. sea-lion flight log). Requires
+  `ctx`; uses metadataFile and open from ctx. Unlike calibration this is not gated
+  to stereo datasets.
+-->
+<script lang="ts">
+import { defineComponent } from 'vue';
+import { importMultiCamContextProp } from './importMultiCamContext';
+
+export default defineComponent({
+  name: 'ImportMultiCamMetadata',
+  props: {
+    ...importMultiCamContextProp,
+  },
+  setup(props) {
+    const {
+      metadataFile,
+      open,
+      clearMetadataFile,
+    } = props.ctx;
+    return {
+      metadataFile,
+      open,
+      clearMetadataFile,
+    };
+  },
+});
+</script>
+
+<template>
+  <v-row
+    no-gutters
+    class="align-center my-3"
+  >
+    <v-text-field
+      label="Metadata File (Optional)"
+      placeholder="Not selected"
+      readonly
+      outlined
+      dense
+      hide-details
+      :value="metadataFile"
+      hint="A .json, .txt, or .csv file passed to pipelines that request it."
+      class="mr-3"
+    />
+    <v-btn
+      v-if="metadataFile"
+      icon
+      class="mr-2"
+      aria-label="Clear metadata file"
+      @click="clearMetadataFile"
+    >
+      <v-icon>mdi-close</v-icon>
+    </v-btn>
+    <v-btn
+      color="primary"
+      @click="open('metadata', 'metadata')"
+    >
+      Choose metadata
+      <v-icon class="ml-2">
+        mdi-file-cog
+      </v-icon>
+    </v-btn>
+  </v-row>
+</template>

client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts

@@ -96,6 +96,9 @@ export function useImportMultiCamDialog(
   const calibrationFile = ref('');
   const lastCalibrationPath = ref('');
   const calibrationAutoDiscoveryFailed = ref(false);
+  // Optional per-dataset metadata file (e.g. sea-lion flight log). Not gated to
+  // stereo; applies to any multicam import.
+  const metadataFile = ref('');
   const datasetName = ref('');
   const subfolderOriginalNames: Ref<Record<string, string>> = ref({});
   const cameraOrder: Ref<string[]> = ref([]);
@@ -656,8 +659,8 @@ export function useImportMultiCamDialog(
   }
 
   async function open(
-    dstype: DatasetType | 'calibration' | 'text',
-    folder: string | 'calibration',
+    dstype: DatasetType | 'calibration' | 'text' | 'metadata',
+    folder: string | 'calibration' | 'metadata',
     directory = false,
   ) {
     const ret = await openFromDisk(dstype, directory || dstype === 'image-sequence');
@@ -670,6 +673,8 @@ export function useImportMultiCamDialog(
         if (saveCalibration) {
           saveCalibration(path);
         }
+      } else if (folder === 'metadata') {
+        metadataFile.value = path;
       } else if (importType.value === 'multi') {
         if (ret.root) {
           folderList.value[folder].sourcePath = ret.root;
@@ -781,6 +786,7 @@ export function useImportMultiCamDialog(
         cameraOrder: orderedCameraKeys.value,
         sourceList,
         calibrationFile: calibrationFile.value,
+        metadataFile: metadataFile.value || undefined,
         type: props.dataType,
       };
       emit('begin-multicam-import', args);
@@ -790,6 +796,7 @@ export function useImportMultiCamDialog(
         sourcePath: keywordFolder.value,
         globList: globList.value,
         calibrationFile: calibrationFile.value,
+        metadataFile: metadataFile.value || undefined,
         type: 'image-sequence',
       };
       emit('begin-multicam-import', args);
@@ -817,6 +824,10 @@ export function useImportMultiCamDialog(
     calibrationFile.value = '';
   }
 
+  function clearMetadataFile() {
+    metadataFile.value = '';
+  }
+
   function applyLastCalibration() {
     if (!lastCalibrationPath.value) {
       return;
@@ -951,5 +962,7 @@ export function useImportMultiCamDialog(
     openTransformFile,
     clearTransformFile,
     clearCalibration,
+    metadataFile,
+    clearMetadataFile,
   };
 }

client/dive-common/constants.ts

@@ -60,6 +60,14 @@ const calibrationFileTypes = [
   'zip',
 ];
 
+// Optional per-dataset metadata file (e.g. sea-lion flight log) that is stored
+// alongside the dataset and handed to opt-in pipelines at run time.
+const metadataFileTypes = [
+  'json',
+  'txt',
+  'csv',
+];
+
 const fileVideoTypes = [
   'mp4',
   'webm',
@@ -182,6 +190,8 @@ const stereoPipelineMarker = 'measurement';
 const calibrationFileMarker = 'calibrationFile';
 /** Girder item meta key marking the JSON camera-rig used for calibration display. */
 const jsonCalibrationFileMarker = 'jsonCalibrationFile';
+/** Girder item meta key marking the optional per-dataset metadata file upload. */
+const metadataFileMarker = 'metadataFile';
 /** Legacy common_stereo category key; never shown in the run-pipeline menu. */
 const hiddenPipelineCategories = ['stereo'];
 /** Pipeline name/category substrings hidden from the web run-pipeline menu. */
@@ -205,6 +215,7 @@ export {
   FPSOptions,
   itemsPerPageOptions,
   calibrationFileTypes,
+  metadataFileTypes,
   fileVideoTypes,
   otherImageTypes,
   otherVideoTypes,
@@ -225,6 +236,7 @@ export {
   stereoPipelineMarker,
   calibrationFileMarker,
   jsonCalibrationFileMarker,
+  metadataFileMarker,
   hiddenPipelineCategories,
   webExcludedPipelineTerms,
   multiCamPipelineMarkers,

client/platform/desktop/backend/native/common.ts

@@ -141,7 +141,7 @@ async function extractPipeMetadata(filePath: string): Promise<PipeMetadata> {
       }
 
       if (inDescription) {
-        if (/^#\s*$/.test(line) || /^#\s*=/.test(line) || /^#\s*(Input|Output|Requires\s+Calibration):/i.test(line) || !line.startsWith('#')) {
+        if (/^#\s*$/.test(line) || /^#\s*=/.test(line) || /^#\s*(Input|Output|Requires\s+Calibration|Metadata\s+File|Image\s+List\s+Keys?):/i.test(line) || !line.startsWith('#')) {
           inDescription = false;
         } else {
           fullDescription += ` ${line.replace(/^#\s*/, '').trim()}`;
@@ -162,6 +162,28 @@ async function extractPipeMetadata(filePath: string): Promise<PipeMetadata> {
         const value = calibrationMatch[1].trim().toLowerCase();
         metadata.requiresCalibration = ['true', 'yes', '1'].includes(value);
       }
+
+      // `# Metadata File: <block>:<key>` opts a pipe in to receiving the
+      // dataset's optional metadata file as a `-s <block>:<key>=<path>` override.
+      const metadataFileMatch = line.match(/^#\s*Metadata\s+File:\s*(.+)/i);
+      if (metadataFileMatch) {
+        const value = metadataFileMatch[1].trim();
+        if (value) {
+          metadata.metadataFileKey = value;
+        }
+      }
+
+      // `# Image List Keys: <k> [k...]` binds the run's input image list(s) (one
+      // per camera; multicam comma-joined) to each key, so pipes (e.g. the
+      // sea-lion registration stabilizer) read the same image list DIVE feeds the
+      // input reader.
+      const imageListMatch = line.match(/^#\s*Image\s+List\s+Keys?:\s*(.+)/i);
+      if (imageListMatch) {
+        const keys = imageListMatch[1].trim().split(/[\s,]+/).filter((k) => k);
+        if (keys.length) {
+          metadata.imageListKeys = keys;
+        }
+      }
     });
     metadata.description = fullDescription.trim() || undefined;
   } catch (error) {
@@ -1498,6 +1520,25 @@ async function finalizeMediaImport(
     jsonMeta.multiCam.calibrationSourcePath = preservedOriginalPath;
   }
 
+  // Store any optional metadata file alongside the media (keeping the original
+  // name). Single imports pass it on the response; multicam imports stash the
+  // source path on jsonMeta.metadataFile during beginMultiCamImport.
+  const metadataSourcePath = args.metadataFileAbsPath || jsonMeta.metadataFile;
+  if (metadataSourcePath) {
+    const resolvedMetadataSource = npath.resolve(metadataSourcePath);
+    const metadataDest = npath.join(
+      projectDirAbsPath,
+      npath.basename(resolvedMetadataSource),
+    );
+    await fs.copy(resolvedMetadataSource, metadataDest);
+    jsonMeta.metadataOriginalName = npath.basename(resolvedMetadataSource);
+    jsonMeta.metadataFile = metadataDest;
+  } else {
+    // Ensure a stale source path never survives when no file was chosen.
+    jsonMeta.metadataFile = undefined;
+    jsonMeta.metadataOriginalName = undefined;
+  }
+
   // Filter all parts of the input based on glob pattern
   if (globPattern && jsonMeta.type === 'image-sequence') {
     const searchPath = jsonMeta.imageListPath || jsonMeta.originalBasePath;

client/platform/desktop/backend/native/multiCamImport.ts

@@ -199,6 +199,9 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise<DesktopMed
       calibration: args.calibrationFile,
       defaultDisplay: args.defaultDisplay,
     },
+    // Stash the metadata source path; finalizeMediaImport copies it into the
+    // project directory and rewrites this to the stored copy's path.
+    metadataFile: args.metadataFile || undefined,
     subType: null,
   };
   if (Object.keys(seedHomographies).length || Object.keys(seedCorrespondences).length) {