feat: add capabilities for spatial layout type (#1138)

* feat: spatial MVP

* chore(editor): spatial examples are the default demo in the editor (#1127)

chore: make the spatial examples the default

* feat: 3D model for a spatial view specified in the layout (#1131)

* feat: minimal join data transform

* chore: add a todo statement

* first working demo

* more work on bigwig join

* first demo to load bigwig and full chromosomes

* all d3 models drawn

* chore: remove console log statements and clean up code

* chore: fix typo in the comment statement

* chore: more clean up

* chore: compact axis

* fix: fix loading json-based track

* feat: grammar extension for 3D layout (#1136)

feat: better grammar extension for 3d vis

* chore: move the 3d examples to the top; use the new grammar in example specs (#1139)

chore: move the 3d examples to the top; use the new grammar in examples

* feat(editor): add a 3D yeast model example displayed in three layout options (#1140)

* feat: add a Yeast model example

* add thumbnail for yeast example

* make the yeast example the default

at least for now

---------

Co-authored-by: David Kouril <[email protected]>

* Clean up basic examples (#1141)

* chore: minimal example

replaced the default (draft) thumbnail, also removed the very early example i used (with linked views) and the thumbnail

* chore: remove the multiple tracks example for now

still needs some implementation to be able to show that

* feat: remove data requirement for 3D tracks (#1142)

* feat: remove data requirement for 3D tracks

* comment statement

* chore: update demo examples

* fix: parsed genomic coordinates shouldn't be NaN even if assembly is not specified (#1143)

* fix: parse genomic coordinates properly even if assembly is incorrectly defined

* remove console logs

* join correctly when there is only a single genomic field

* Example: three different ways of setting color (#1144)

* feat: add example of different coloring strategies

constant vs nominal vs quantitative

* chore: remove data fields in tracks since it's not needed anymore

* chore: change the default model for minimal to stevens

also reverted back to being really 'minimal'

* fix: pass the mark option from spec

* feat: density example (#1145)

* feat(editor): example of composing multiple tracks within one spatial view (#1147)

* wip: combining multiple tracks within a single spatial view

very very bad so far, just a first checkpoint toward something sorta working

* feat: add a new example for superimposition

* wip: multiple overlaid track

* feat: (wip)example for composing multiple tracks within one spatial view

touches the join implementation a bit

* chore: remove computation of parent views

* chore: clean up of unused properties

* chore: clean up (log, unused var)

* chore: delete commented line

* chore: delete log and commented out line

* feat(editor): add example using models from Tan 2018 (#1148)

* feat(editor): Tan example highlights using sequential color (#1149)

* format

* more format/eslint

* format: remove unused vars

* dirty assert options as SpatialTrackOptions

* refactor: somewhat better type check when fetching column from tables

* refactor: getting rid of squiggly lines

* refactor: typing of the handleColor function

* refactor: type errors in handleSizeField

* wip

* temporary fix: doesn't make sense now but new uchimata will fix it

* refactor: use new library name and update api usage

renamed the underlying library from `chromospace` to `uchimata`

* (wip)fix: type errors around spatial views/tracks

* suppress warning about `ProcessedSpatialTrack`

Same thing is used for GoslingTrack. There's probably a better solution

* assert xScale when using certain transforms

also moved the assert function to separate file

* fix import path

* cleaning up some properties

* rename ChromospaceTrack to SpatialTrack

* mark a todo for later

* regenerate json schema

* add Encoding in the SpatialTrack type

* add data to spatialtrack

* wip: copying over some properties

some done by claude

* wip(broken): sorted out the types

for the most part, the typing should be fine. next will actually make it work again

* working version without type errors

* docs

* regenerate schema

* cleaning up the changes

* remove console logs

* run prettier on package.json

was getting some weird merge conflicts

* format

* Update editor/example/json-spec/3d-yeast-model.ts

Co-authored-by: SEHI L'YI <[email protected]>

* Update editor/example/json-spec/3d-yeast-model.ts

Co-authored-by: SEHI L'YI <[email protected]>

---------

Co-authored-by: David Kouril <[email protected]>

Author: sehilyi

demo/gosling-component.tsx

@@ -85,6 +85,7 @@ export function GoslingComponent(props: GoslingComponentProps) {
         ></div>
     );
 }
+
 /**
  * This is the main function. It takes a Gosling spec and renders it using the PixiManager
  */

demo/renderer/dataFetcher.ts

@@ -22,7 +22,9 @@ export function getDataFetcher(spec: SingleTrack | OverlaidTrack, urlToFetchOpti
 
     if (typeof data === 'undefined') {
         console.warn('No data in the track spec', spec);
-        return;
+        // A data object can be missing in 3D tracks. In this case, just pass an empty data.
+        // The 3d track can still use the 3D genome model for visual encoding.
+        return new JsonDataFetcher({ type: 'json', values: [], assembly: spec.assembly ?? 'unknown' });
     }
 
     const { type } = data;

demo/renderer/main.ts

@@ -18,6 +18,8 @@ import type { UrlToFetchOptions } from 'src/compiler/compile';
 import type { Tile } from '@higlass/services';
 import type { DataFetcher } from '@higlass/datafetcher';
 import type { OverlaidTrack, SingleTrack } from '@gosling-lang/gosling-schema';
+import { createSpatialTrack, type SpatialTrackOptions } from '../../src/tracks/spatial-track/spatial-track';
+import type { CsvDataFetcherClass } from 'src/data-fetchers/csv/csv-data-fetcher';
 
 /**
  * Takes a list of track definitions and linkedEncodings and renders them
@@ -146,6 +148,23 @@ export function renderTrackDefs(
             const dummyPlot = new DummyTrack(dummyOptions, pixiManager.makeContainer(boundingBox).overlayDiv);
             plotDict[trackId] = dummyPlot;
         }
+        // Add a new track type for `spatial` layout (rendered via uchimata)
+        if (type === TrackType.Spatial) {
+            // Even though uchimata doesn't use PixiJS, we can use the PixiManager to create a div container that the canvas can be placed into.
+            // In the final version, we would probably want uchimata to use an existing canvas element (to limit the creation of new elements).
+            // But for now this gets the job done.
+            const container = pixiManager.makeContainer(boundingBox).overlayDiv;
+            const spatialTrackOptions = options as SpatialTrackOptions; //~ TODO: properly assert the type!
+            console.warn('!@$!#%@# detected spatial track !@#$!#%@#');
+            if (spatialTrackOptions.spec.data) {
+                // Ensure to pull all data needed
+                if ('sampleLength' in spatialTrackOptions.spec.data) {
+                    spatialTrackOptions.spec.data.sampleLength = 30000;
+                }
+            }
+            const datafetcher = getDataFetcher(spatialTrackOptions.spec, urlToFetchOptions);
+            createSpatialTrack(spatialTrackOptions, datafetcher as CsvDataFetcherClass, container);
+        }
     });
     return plotDict;
 }

demo/track-def/main.ts

@@ -6,6 +6,7 @@ import { type BrushLinearTrackOptions } from '@gosling-lang/brush-linear';
 import type { TrackInfo } from '../../src/compiler/bounding-box';
 import type { CompleteThemeDeep } from '../../src/core/utils/theme';
 import type { GoslingTrackOptions } from '../../src/tracks/gosling-track/gosling-track';
+import type { SpatialTrackOptions } from 'src/tracks/spatial-track/spatial-track';
 
 import { proccessTextHeader } from './text';
 import { processHeatmapTrack, isHeatmapTrack } from './heatmap';
@@ -25,7 +26,8 @@ export enum TrackType {
     Axis,
     BrushLinear,
     BrushCircular,
-    Heatmap
+    Heatmap,
+    Spatial
 }
 
 /**
@@ -39,6 +41,7 @@ interface TrackOptionsMap {
     [TrackType.BrushLinear]: BrushLinearTrackOptions;
     [TrackType.BrushCircular]: BrushCircularTrackOptions;
     [TrackType.Heatmap]: HeatmapTrackOptions;
+    [TrackType.Spatial]: SpatialTrackOptions;
 }
 
 /**
@@ -81,6 +84,19 @@ export function createTrackDefs(trackInfos: TrackInfo[], theme: Required<Complet
             // We have a dummy track
             const dummyTrackDefs = processDummyTrack(track, boundingBox, theme);
             trackDefs.push(...dummyTrackDefs);
+        } else if ('layout' in track && track.layout === 'spatial') {
+            // We have a 3D track
+            const trackDef: TrackDef<SpatialTrackOptions> = {
+                type: TrackType.Spatial,
+                trackId: track.id,
+                boundingBox,
+                options: {
+                    // @ts-expect-error At this point, the spec is processed
+                    spec: track,
+                    processedSpec: track
+                }
+            };
+            trackDefs.push(trackDef);
         } else {
             // We have a gosling track
             const goslingAxisDefs = processGoslingTrack(track, boundingBox, theme);

demo/track-def/types.ts

@@ -13,16 +13,19 @@ import type { DataDeep, Assembly, DummyTrackStyle, Mark, X, Y } from '@gosling-l
  */
 
 /** A Track after it has been compiled */
-export type ProcessedTrack = ProcessedLinearTrack | ProcessedCircularTrack | ProcessedDummyTrack;
-// TODO: This should be much better expanded to include all the properties that are actually present.
+export type ProcessedTrack =
+    | ProcessedLinearTrack
+    | ProcessedCircularTrack
+    | ProcessedDummyTrack
+    | ProcessedSpatialTrack;
 /** All tracks potentially have these properties */
 export interface ProcessedTrackBase {
-    layout?: 'linear' | 'circular';
+    layout?: 'linear' | 'circular' | 'spatial';
     id: string;
     height: number;
     width: number;
     static: boolean;
-    mark?: string;
+    mark?: Mark;
     orientation: 'horizontal' | 'vertical';
     title?: string;
     subtitle?: string;
@@ -51,6 +54,19 @@ export type ProcessedCircularTrack = ProcessedTrackBase & {
     innerRadius: number;
 };
 
+//~ DK: AFAIK, this needs to loosely fit definitions in Track.
+//~ it is force casted via `as ProcessedTrack`, so there's no strict type checking.
+export type ProcessedSpatialTrack = ProcessedTrackBase & {
+    layout: 'spatial';
+    spatial: {
+        x: string;
+        y: string;
+        z: string;
+        chr: string;
+        coord: string;
+    };
+};
+
 export type ProcessedDummyTrack = (ProcessedLinearTrack | ProcessedCircularTrack) & {
     type: 'dummy-track';
     style?: DummyTrackStyle;

docs/spatial-layout.md

@@ -0,0 +1,68 @@
+# 3D genome structures in Gosling
+
+This file documents the changes in the specification and gosling.js library to
+support visualizing 3D structures of genomes.
+
+In principle, there are two ways to define a spatial view:
+1. on the **view level**
+2. on the **track level**
+
+## view-level specification
+We can supply a 3D model data file and bind it to spatial layout properties in
+the definition of view's `layout`. This has two advantages: 1) there is no need
+to repeat the definitions of a model in each track, and 2) this enables to
+merge two data files (one to define the 3D model, and another with a genomic
+signal).
+
+```javascript
+{
+  views: [
+    {
+      layout: {
+        type: "spatial",
+        model: {
+          type: "csv",
+          url: "https://.../full-model.csv",
+          xyz: ["x", "y", "z"],
+          chromosome: "chr",
+          position: "coord"
+        }
+      },
+      tracks: [
+         { mark: "sphere" },
+         { mark: "box" },
+      ]
+    }
+  ]
+}
+```
+
+The other method uses simply specifying layout as `spatial` and using the data
+in a track for defining the model:
+```javascript
+{
+  views: [
+    {
+      layout: "spatial",
+      tracks: [
+         {
+            data: { type: "csv", url: "https://.../full-model.csv" },
+            spatial: {
+                  x: "xfield",
+                  y: "yfield",
+                  z: "zfield"
+                  chr: "chrfield",
+                  coord: "positionfield",
+            },
+            mark: "sphere",
+         }
+      ]
+    }
+  ]
+}
+```
+This doesn't allow for combining the structural data with any other genomic
+formats, unless the genomic data are integrated in the csv already.
+
+Internally, the first method is converted into the second one. The model data
+file from the `layout` is pushed down to a dataTransform.

editor/example/index.ts

@@ -29,6 +29,7 @@ import JS_SPEC_ISLANDVIEWER from './spec/islandviewer.ts?raw';
 import { spec as JSON_SPEC_ISLANDVIEWER } from './spec/islandviewer';
 
 export type ExampleGroup =
+    | 'Spatial Layout'
     | 'Visual Encoding'
     | 'Mouse Events'
     | 'Semantic Zooming'
@@ -44,6 +45,10 @@ export const ExampleGroups: {
     name: ExampleGroup;
     description: string;
 }[] = [
+    {
+        name: 'Spatial Layout',
+        description: 'Visualizations using 3D genome models'
+    },
     {
         name: 'Visual Encoding',
         description:
@@ -423,6 +428,43 @@ export const editorExampleObj: {
         spec: JsonExampleSpecs.EX_SPEC_PERF_ALIGNMENT,
         image: THUMBNAILS.PERF_ALIGNMENT
     },
+    SPATIAL_LAYOUT_MIN: {
+        group: 'Spatial Layout',
+        name: 'Minimal Example',
+        spec: JsonExampleSpecs.EX_SPEC_SPATIAL_MINIMAL,
+        image: THUMBNAILS.SPATIAL_MINIMAL
+    },
+    SPATIAL_LAYOUT_COLORS: {
+        group: 'Spatial Layout',
+        name: 'Coloring Strategies',
+        spec: JsonExampleSpecs.EX_SPEC_SPATIAL_COLOR,
+        image: THUMBNAILS.SPATIAL_COLOR
+    },
+    SPATIAL_LAYOUT_DENSITY: {
+        group: 'Spatial Layout',
+        name: 'Density',
+        spec: JsonExampleSpecs.EX_SPEC_SPATIAL_DENSITY,
+        image: THUMBNAILS.SPATIAL_DENSITY
+    },
+    SPATIAL_LAYOUT_SUPERIMP: {
+        group: 'Spatial Layout',
+        name: 'Multiple tracks',
+        spec: JsonExampleSpecs.EX_SPEC_SPATIAL_MULTIPLE_TRACKS,
+        image: THUMBNAILS.SPATIAL_MULTIPLE_TRACKS
+    },
+    SPATIAL_LAYOUT_TAN_COMP: {
+        group: 'Spatial Layout',
+        name: 'Human Cell Model (Tan et al., Science 2018): Comparison',
+        spec: JsonExampleSpecs.EX_SPEC_SPATIAL_TAN_COMP,
+        image: THUMBNAILS.SPATIAL_TAN
+    },
+    SPATIAL_LAYOUT_3D_YEAST_MODEL: {
+        group: 'Spatial Layout',
+        name: '3D Yeast Model (Nature 2010)',
+        spec: JsonExampleSpecs.EX_SPEC_3D_YEAST_MODEL,
+        image: THUMBNAILS.SPATIAL_YEAST,
+        forceShow: true
+    },
     CORCES_ET_AL: {
         group: 'Coordinated Multiple Views',
         name: 'Corces et al. 2020',