Add a video plot that supports video and animated image files

.github/workflows/osv-scanner.yaml

@@ -21,18 +21,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
       - name: Run OSV Scanner (human-readable)
         continue-on-error: true
-        uses: google/osv-scanner-action/osv-scanner-action@v2.3.2
+        uses: google/osv-scanner-action/osv-scanner-action@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8
         with:
           scan-args: |
             --lockfile=pnpm-lock.yaml
 
       - name: Run OSV Scanner (JSON)
         continue-on-error: true
-        uses: google/osv-scanner-action/osv-scanner-action@v2.3.2
+        uses: google/osv-scanner-action/osv-scanner-action@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8
         with:
           scan-args: |
             --format=json

client/component/src/VideoPlot.tsx

@@ -0,0 +1,246 @@
+import {
+  DefaultInteractions,
+  type ModifierKey,
+  ResetZoomButton,
+  TooltipMesh,
+  VisMesh,
+  VisCanvas,
+  getVisDomain,
+  FloatingControl,
+} from '@h5web/lib';
+import { useEffect, useMemo, useState, type ReactElement } from 'react';
+import { useFrame, useThree } from '@react-three/fiber';
+import { useTexture, useVideoTexture } from '@react-three/drei';
+
+import SelectionComponent from './SelectionComponent';
+import type { PlotBaseProps } from './models';
+import { createInteractionsConfig, InteractionModeType } from './utils';
+import {
+  PlotCustomizationContextProvider,
+  usePlotCustomizationContext,
+} from './PlotCustomizationContext';
+import { AnyToolbar } from './PlotToolbar';
+
+/**
+ * Props for the `VideoPlot` component.
+ */
+interface VideoPlotProps extends PlotBaseProps {
+  /** video source url */
+  sourceURL: string;
+  /** if true, source is an animated image (MJPEG, animated GIF, etc) */
+  isImage?: boolean;
+}
+
+function ImageMeshMaterial(props: {
+  sourceUrl: string;
+  play: boolean;
+  setSize: (size: [number, number]) => void;
+}) {
+  const { setFrameloop } = useThree();
+  const texture = useTexture(props.sourceUrl);
+  const data = texture.source.data as HTMLImageElement;
+  const height = data.height;
+  const width = data.width;
+
+  useEffect(() => {
+    props.setSize([height, width]);
+  }, [height, width, props]);
+
+  useEffect(() => {
+    return () => {
+      setFrameloop('demand');
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  useEffect(() => {
+    // dial down frame loop when no more frames
+    if (data.complete) {
+      setFrameloop('demand');
+    }
+  }, [data.complete, setFrameloop]);
+
+  useEffect(() => {
+    setFrameloop(props.play ? 'always' : 'demand');
+  }, [props.play, setFrameloop]);
+
+  // eslint-disable-next-line react-hooks/immutability
+  useFrame(() => (texture.needsUpdate = props.play));
+
+  // eslint-disable-next-line react/no-unknown-property
+  return <meshBasicMaterial map={texture} />;
+}
+
+function VideoMeshMaterial(props: {
+  sourceUrl: string;
+  play: boolean;
+  setSize: (size: [number, number]) => void;
+}) {
+  const { setFrameloop } = useThree();
+  const texture = useVideoTexture(props.sourceUrl);
+  const data = texture.source.data;
+
+  useEffect(() => {
+    data.pause(); // start paused
+    return () => {
+      setFrameloop('demand');
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const height = data.videoHeight;
+  const width = data.videoWidth;
+  useEffect(() => {
+    props.setSize([height, width]);
+  }, [height, width, props]);
+
+  useEffect(() => {
+    if (props.play) {
+      data.play().catch((e) => {
+        console.log('Could not play: %s', e);
+      });
+    } else {
+      data.pause();
+    }
+    setFrameloop(props.play ? 'always' : 'demand');
+  }, [data, props.play, setFrameloop]);
+
+  // eslint-disable-next-line react/no-unknown-property
+  return <meshBasicMaterial map={texture} />;
+}
+
+function PlayButton(props: { clickPlay: () => void; text: string }) {
+  return (
+    <FloatingControl>
+      <button type="button" onClick={() => props.clickPlay()}>
+        <span>{props.text}</span>
+      </button>
+    </FloatingControl>
+  );
+}
+
+export function VideoVisCanvas(props: VideoPlotProps) {
+  const {
+    title,
+    showGrid,
+    xCustomDomain,
+    setXCustomDomain,
+    xDomain,
+    xLabel,
+    yCustomDomain,
+    setYCustomDomain,
+    yDomain,
+    yLabel,
+    mode,
+    batonProps,
+    canSelect,
+    selectionMax,
+    selectionType,
+    updateSelection,
+    selections,
+  } = usePlotCustomizationContext();
+  const { sourceURL, isImage = false } = { ...props };
+  const [isPlaying, setPlaying] = useState(false);
+
+  const interactionsConfig = createInteractionsConfig(mode);
+  const tooltipText = (x: number, y: number): ReactElement<string> => {
+    return (
+      <p>
+        {x.toPrecision(8)}, {y.toPrecision(8)}
+      </p>
+    );
+  };
+
+  const [size, setSize] = useState([1, 1]);
+
+  useEffect(() => {
+    setXCustomDomain([0, size[1]]);
+    setYCustomDomain([0, size[0]]);
+  }, [size, setXCustomDomain, setYCustomDomain]);
+  const xVisDomain = useMemo(() => {
+    return getVisDomain(xCustomDomain, xDomain);
+  }, [xCustomDomain, xDomain]);
+
+  const yVisDomain = useMemo(() => {
+    return getVisDomain(yCustomDomain, yDomain);
+  }, [yCustomDomain, yDomain]);
+
+  return (
+    <VisCanvas
+      title={title ?? ''}
+      aspect={'equal'}
+      abscissaConfig={{
+        visDomain: xVisDomain,
+        showGrid,
+        isIndexAxis: true,
+        label: xLabel,
+        nice: true,
+      }}
+      ordinateConfig={{
+        visDomain: yVisDomain,
+        showGrid,
+        isIndexAxis: true,
+        label: yLabel,
+        nice: true,
+      }}
+    >
+      <DefaultInteractions {...interactionsConfig} />
+      <ResetZoomButton />
+      <PlayButton
+        clickPlay={() => setPlaying((p) => !p)}
+        text={isPlaying ? 'Pause' : 'Play'}
+      />
+      <TooltipMesh renderTooltip={tooltipText} />
+      <VisMesh>
+        {isImage ? (
+          <ImageMeshMaterial
+            sourceUrl={sourceURL}
+            play={isPlaying}
+            setSize={setSize}
+          />
+        ) : (
+          <VideoMeshMaterial
+            sourceUrl={sourceURL}
+            play={isPlaying}
+            setSize={setSize}
+          />
+        )}
+      </VisMesh>
+      {canSelect && (
+        <SelectionComponent
+          modifierKey={[] as ModifierKey[]}
+          batonProps={batonProps}
+          disabled={mode !== InteractionModeType.selectRegion}
+          selectionMax={selectionMax}
+          selectionType={selectionType}
+          updateSelection={updateSelection}
+          selections={selections}
+        />
+      )}
+    </VisCanvas>
+  );
+}
+
+/**
+ * Render a video plot.
+ * @param {VideoPlotProps} props - The component props.
+ * @returns {JSX.Element} The rendered component.
+ */
+function VideoPlot(props: VideoPlotProps) {
+  return (
+    <div
+      style={{
+        display: 'grid',
+        position: 'relative',
+      }}
+    >
+      <PlotCustomizationContextProvider {...props}>
+        <AnyToolbar>{props.customToolbarChildren}</AnyToolbar>
+        <VideoVisCanvas {...props} />
+      </PlotCustomizationContextProvider>
+    </div>
+  );
+}
+
+export default VideoPlot;
+export type { VideoPlotProps };

client/component/src/index.ts

@@ -21,6 +21,9 @@ export type { ImageData, ImagePlotProps } from './ImagePlot';
 export { default as LinePlot } from './LinePlot';
 export type { LineData, LineParams, LinePlotProps } from './LinePlot';
 
+export { default as VideoPlot } from './VideoPlot';
+export type { VideoPlotProps } from './VideoPlot';
+
 export {
   PlotCustomizationContextProvider,
   usePlotCustomizationContext,

storybook/.storybook/main.ts

@@ -11,6 +11,7 @@ const config: StorybookConfig = {
       from: '../../typedocs',
       to: 'typedocs',
     },
+    '../public',
   ],
 
   addons: ['@storybook/addon-links', '@storybook/addon-docs'],

storybook/.storybook/preview.ts

@@ -21,6 +21,7 @@ const preview: Preview = {
             'Scatter',
             'Surface',
             'Table',
+            'Video',
             'ConnectedPlot',
           ],
           'Modals',

storybook/stories/Video.stories.tsx

@@ -0,0 +1,23 @@
+import type { Meta, StoryObj } from '@storybook/react-vite';
+import { VideoPlot } from '@diamondlightsource/davidia';
+
+const meta: Meta<typeof VideoPlot> = {
+  title: 'Plots/Video',
+  component: VideoPlot,
+  tags: ['autodocs'],
+};
+
+export default meta;
+
+export const Heatmap: StoryObj<typeof VideoPlot> = {
+  args: {
+    plotConfig: {
+      title: 'Sample Video Plot',
+      xLabel: 'x-axis',
+      yLabel: 'y-axis',
+    },
+    // sourceURL: "http://localhost:8080/stream.mjpeg",
+    // isImage: true,
+    sourceURL: 'Jellyfish_1080_10s_2MB.webm',
+  },
+};