From f0c021675a39ec686e7f37ade94f3cb5b2699596 Mon Sep 17 00:00:00 2001 From: Paul Crossley Date: Wed, 20 Nov 2024 11:32:57 +0000 Subject: [PATCH 1/3] init --- .../projects/modelling/components/index.ts | 2 ++ .../components/natmap_soil_component.ts | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 app/javascript/projects/modelling/components/natmap_soil_component.ts diff --git a/app/javascript/projects/modelling/components/index.ts b/app/javascript/projects/modelling/components/index.ts index 3fe1e8a..9f53b17 100644 --- a/app/javascript/projects/modelling/components/index.ts +++ b/app/javascript/projects/modelling/components/index.ts @@ -40,6 +40,7 @@ import { SoilComponent } from "./soil_component" import { SegmentComponent } from "./segment_component" import { KewSamplesComponent } from "./kew_samples_component" import { InterpolationComponent } from "./interpolation_component" +import { NatmapSoilComponent } from "./natmap_soil_component" export interface ProjectProperties { extent: Extent @@ -58,6 +59,7 @@ export function createDefaultComponents(saveMapLayer: SaveMapLayer, saveModel: S // Team permissions restrict some components. Add them here. if (permissions.DefraHedgerows) restrictedComponents.push(new HedgerowComponent(projectProps)) if (permissions.KewSamples) restrictedComponents.push(new KewSamplesComponent(projectProps)) + if (true) restrictedComponents.push(new NatmapSoilComponent(projectProps)) // Freely available components here. const components : BaseComponent[] = [ diff --git a/app/javascript/projects/modelling/components/natmap_soil_component.ts b/app/javascript/projects/modelling/components/natmap_soil_component.ts new file mode 100644 index 0000000..962e67e --- /dev/null +++ b/app/javascript/projects/modelling/components/natmap_soil_component.ts @@ -0,0 +1,20 @@ +import { NodeData, WorkerInputs, WorkerOutputs } from "rete/types/core/data" +import { ProjectProperties } from "." +import { Node, Output } from "rete" +import { BaseComponent } from "./base_component" + + + +export class NatmapSoilComponent extends BaseComponent { + + constructor(projectProps: ProjectProperties) { + super("NATMAP Soil") + this.category = "Inputs" + } + + async builder(node: Node) { + } + + async worker(node: NodeData, inputs: WorkerInputs, outputs: WorkerOutputs, ...args: unknown[]) { + } +} \ No newline at end of file From 1f19812b8f18cef0971ab36ef8853255af8c6f89 Mon Sep 17 00:00:00 2001 From: Paul Crossley Date: Thu, 21 Nov 2024 13:45:01 +0000 Subject: [PATCH 2/3] natmap component & minor refactor to tile_grid --- .../components/natmap_soil_component.ts | 183 +++++++++++++++++- .../projects/modelling/model_retrieval.ts | 24 +++ .../projects/modelling/tile_grid.ts | 19 ++ 3 files changed, 225 insertions(+), 1 deletion(-) diff --git a/app/javascript/projects/modelling/components/natmap_soil_component.ts b/app/javascript/projects/modelling/components/natmap_soil_component.ts index 962e67e..565cfb4 100644 --- a/app/javascript/projects/modelling/components/natmap_soil_component.ts +++ b/app/javascript/projects/modelling/components/natmap_soil_component.ts @@ -1,20 +1,201 @@ import { NodeData, WorkerInputs, WorkerOutputs } from "rete/types/core/data" import { ProjectProperties } from "." -import { Node, Output } from "rete" +import { Node, Output, Socket } from "rete" import { BaseComponent } from "./base_component" +import { SelectControlOptions } from "../controls/select" +import { numericDataSocket } from "../socket_types" +import { retrieveWFSData } from "../model_retrieval" +import { Feature } from "ol" +import { Geometry } from "ol/geom" +import { BooleanTileGrid, NumericTileGrid } from "../tile_grid" +import { createXYZ } from "ol/tilegrid" +import { maskFromExtentAndShape } from "../bounding_box" +interface NatmapSoilOptions extends SelectControlOptions { + key: string + socket: Socket +} +const natmap_outputs : NatmapSoilOptions[] = [ + { + id: 0, + name: 'MIN_STK_10', + key: 'MIN_STK_10', + socket: numericDataSocket + }, + { + id: 1, + name: 'MAX_STK_10', + key: 'MAX_STK_10', + socket: numericDataSocket + }, + { + id: 2, + name: 'MIN_STK_15', + key: 'MIN_STK_15', + socket: numericDataSocket + }, + { + id: 3, + name: 'MAX_STK_15', + key: 'MAX_STK_15', + socket: numericDataSocket + }, + { + id: 4, + name: 'MIN_STK_30', + key: 'MIN_STK_30', + socket: numericDataSocket + }, + { + id: 5, + name: 'MAX_STK_30', + key: 'MAX_STK_30', + socket: numericDataSocket + }, + { + id: 6, + name: 'AV_STK_30', + key: 'AV_STK_30', + socket: numericDataSocket + }, + { + id: 7, + name: 'AV_STK_100', + key: 'AV_STK_100', + socket: numericDataSocket + }, + { + id: 8, + name: 'AV_STK_150', + key: 'AV_STK_150', + socket: numericDataSocket + }, + { + id: 9, + name: 'AV_OC_30', + key: 'AV_OC_30', + socket: numericDataSocket + }, + { + id: 10, + name: 'MIN_OC_30', + key: 'MIN_OC_30', + socket: numericDataSocket + }, + { + id: 11, + name: 'MAX_OC_30', + key: 'MAX_OC_30', + socket: numericDataSocket + }, + { + id: 12, + name: 'AV_OC_100', + key: 'AV_OC_100', + socket: numericDataSocket + }, + { + id: 13, + name: 'MIN_OC_100', + key: 'MIN_OC_100', + socket: numericDataSocket + }, + { + id: 14, + name: 'MAX_OC_100', + key: 'MAX_OC_100', + socket: numericDataSocket + }, + { + id: 15, + name: 'AV_OC_150', + key: 'AV_OC_150', + socket: numericDataSocket + }, + { + id: 16, + name: 'MIN_OC_150', + key: 'MIN_OC_150', + socket: numericDataSocket + }, + { + id: 17, + name: 'MAX_OC_150', + key: 'MAX_OC_150', + socket: numericDataSocket + } + +] + +function applyFeaturesToGrid(features: Feature[], projectProps: ProjectProperties, prop: string, mask: BooleanTileGrid) : NumericTileGrid { + + const tileGrid = createXYZ() + const outputTileRange = tileGrid.getTileRangeForExtentAndZ(projectProps.extent, projectProps.zoom) + const grid = new NumericTileGrid( + projectProps.zoom, + outputTileRange.minX, + outputTileRange.minY, + outputTileRange.getWidth(), + outputTileRange.getHeight() + ) + + for (let feature of features) { + + const geom = feature.getGeometry() + if (geom === undefined) { continue } + + const val = feature.get(prop) + if (val === undefined) { continue } + + const featureTileRange = tileGrid.getTileRangeForExtentAndZ( + geom.getExtent(), + projectProps.zoom + ) + + grid.iterateOverTileRange(featureTileRange, (x, y) => { + const center = tileGrid.getTileCoordCenter([projectProps.zoom, x, y]) + + if (geom.intersectsCoordinate(center)) { + grid.set(x, y, mask.get(x, y) ? val : NaN) + } + } + ) + } + + return grid +} export class NatmapSoilComponent extends BaseComponent { + projectProps: ProjectProperties + cachedFeatures: Feature[] + cachedOutputs: Map constructor(projectProps: ProjectProperties) { super("NATMAP Soil") this.category = "Inputs" + this.projectProps = projectProps + this.cachedFeatures = [] + this.cachedOutputs = new Map() } async builder(node: Node) { + natmap_outputs.forEach(opt => { + node.addOutput(new Output(opt.key, opt.name, opt.socket)) + }) } async worker(node: NodeData, inputs: WorkerInputs, outputs: WorkerOutputs, ...args: unknown[]) { + + this.cachedFeatures = this.cachedFeatures.length === 0 ? await retrieveWFSData('cranfield_soil:NATMAPcarbon', this.projectProps) : this.cachedFeatures + + const mask = await maskFromExtentAndShape(this.projectProps.extent, this.projectProps.zoom, this.projectProps.maskLayer, this.projectProps.maskCQL, this.projectProps.mask) + + natmap_outputs.filter(opt => node.outputs[opt.key].connections.length > 0).forEach(opt => { + const res = this.cachedOutputs.has(opt.key) ? this.cachedOutputs.get(opt.key) : applyFeaturesToGrid(this.cachedFeatures, this.projectProps, opt.key, mask) + this.cachedOutputs.set(opt.key, res as NumericTileGrid) + outputs[opt.key] = res + }) + } } \ No newline at end of file diff --git a/app/javascript/projects/modelling/model_retrieval.ts b/app/javascript/projects/modelling/model_retrieval.ts index 2e581fe..8ab4f44 100644 --- a/app/javascript/projects/modelling/model_retrieval.ts +++ b/app/javascript/projects/modelling/model_retrieval.ts @@ -3,6 +3,30 @@ import * as GeoTIFF from 'geotiff/dist-browser/geotiff' import { Extent } from 'ol/extent' import { bboxFromExtent } from './bounding_box' +import { ProjectProperties } from './components' +import { GeoJSON } from "ol/format" +import { Geometry } from 'ol/geom' +import { Feature } from 'ol' + +export async function retrieveWFSData(source: string, projectProps: ProjectProperties) : Promise[]> { + + const response = await fetch( + "https://landscapes.wearepal.ai/geoserver/wfs?" + + new URLSearchParams( + { + outputFormat: 'application/json', + request: 'GetFeature', + typeName: source, + srsName: 'EPSG:3857', + bbox : bboxFromExtent(projectProps.extent), + } + ) + ) + + const features = new GeoJSON().readFeatures(await response.json()) + + return features +} // Returns a GeoTIFF object from a WMS server. Useful for some categorical/boolean data but may be susceptible to data loss. Fatest option usually export async function retrieveModelData(extent: Extent, source: string, tileRange: any, style?: string) { diff --git a/app/javascript/projects/modelling/tile_grid.ts b/app/javascript/projects/modelling/tile_grid.ts index a452e19..6e62a2d 100644 --- a/app/javascript/projects/modelling/tile_grid.ts +++ b/app/javascript/projects/modelling/tile_grid.ts @@ -3,6 +3,7 @@ import { Extent } from "ol/extent" import { createXYZ } from "ol/tilegrid" import { registerSerializer } from "threads" import { getMedianCellSize } from "./components/cell_area_component" +import { TileRange } from "ol" function validateZoom(zoom: number) { if (!( @@ -244,12 +245,30 @@ export class NumericTileGrid extends TileGrid { } iterate(callback: (x: number, y: number, value: number) => void) { + const { x, y, width, height } = this for (let i = x; i < x + width; i++) { for (let j = y; j < y + height; j++) { callback(i, j, this.get(i, j)) } } + + } + + iterateOverTileRange(range: TileRange, callback: (x: number, y: number, value: number) => void) { + + const { x, y, width, height } = this + const minX = Math.max(x, range.minX) + const maxX = Math.min(x + width, range.maxX) + const minY = Math.max(y, range.minY) + const maxY = Math.min(y + height, range.maxY) + + for (let i = minX; i < maxX; i++) { + for (let j = minY; j < maxY; j++) { + callback(i, j, this.get(i, j)) + } + } + } get(x: number, y: number, zoom = this.zoom): number { From 69786ce2f062753f9bfee9602013bc78b7882071 Mon Sep 17 00:00:00 2001 From: Paul Crossley Date: Thu, 21 Nov 2024 17:03:22 +0000 Subject: [PATCH 3/3] permissions for natmap --- .../controllers/projects_controller.tsx | 5 +- .../projects/modelling/components/index.ts | 2 +- .../components/natmap_soil_component.ts | 91 +++++++++++-------- app/javascript/projects/project_editor.tsx | 1 + app/models/project.rb | 8 ++ app/views/projects/show.html.erb | 1 + .../20241121163637_add_natmap_permission.rb | 9 ++ db/schema.rb | 2 +- 8 files changed, 80 insertions(+), 39 deletions(-) create mode 100644 db/migrate/20241121163637_add_natmap_permission.rb diff --git a/app/javascript/controllers/projects_controller.tsx b/app/javascript/controllers/projects_controller.tsx index 251451f..f1078cd 100644 --- a/app/javascript/controllers/projects_controller.tsx +++ b/app/javascript/controllers/projects_controller.tsx @@ -14,6 +14,7 @@ export default class extends Controller { projectDefraHedgerowPermission: Boolean, projectKewRgb25cmPermission: Boolean, projectKewSamplesPermission: Boolean, + projectNatmapSoilPermission: Boolean, projectExtents: Array, backButtonPath: String, dbModels: Object @@ -26,6 +27,7 @@ export default class extends Controller { declare readonly projectDefraHedgerowPermissionValue: boolean declare readonly projectKewRgb25cmPermissionValue: boolean declare readonly projectKewSamplesPermissionValue: boolean + declare readonly projectNatmapSoilPermissionValue: boolean declare readonly projectExtentsValue: Array declare readonly backButtonPathValue: string declare readonly dbModelsValue: DBModels @@ -43,7 +45,8 @@ export default class extends Controller { { DefraHedgerows: this.projectDefraHedgerowPermissionValue, KewRgb25cm: this.projectKewRgb25cmPermissionValue, - KewSamples: this.projectKewSamplesPermissionValue + KewSamples: this.projectKewSamplesPermissionValue, + NATMAPSoil: this.projectNatmapSoilPermissionValue } } teamExtents={this.projectExtentsValue as TeamExtentData[]} diff --git a/app/javascript/projects/modelling/components/index.ts b/app/javascript/projects/modelling/components/index.ts index 9f53b17..8d9dee3 100644 --- a/app/javascript/projects/modelling/components/index.ts +++ b/app/javascript/projects/modelling/components/index.ts @@ -59,7 +59,7 @@ export function createDefaultComponents(saveMapLayer: SaveMapLayer, saveModel: S // Team permissions restrict some components. Add them here. if (permissions.DefraHedgerows) restrictedComponents.push(new HedgerowComponent(projectProps)) if (permissions.KewSamples) restrictedComponents.push(new KewSamplesComponent(projectProps)) - if (true) restrictedComponents.push(new NatmapSoilComponent(projectProps)) + if (permissions.NATMAPSoil) restrictedComponents.push(new NatmapSoilComponent(projectProps)) // Freely available components here. const components : BaseComponent[] = [ diff --git a/app/javascript/projects/modelling/components/natmap_soil_component.ts b/app/javascript/projects/modelling/components/natmap_soil_component.ts index 565cfb4..8ba7e66 100644 --- a/app/javascript/projects/modelling/components/natmap_soil_component.ts +++ b/app/javascript/projects/modelling/components/natmap_soil_component.ts @@ -14,116 +14,135 @@ import { maskFromExtentAndShape } from "../bounding_box" interface NatmapSoilOptions extends SelectControlOptions { key: string socket: Socket + unit: string } const natmap_outputs : NatmapSoilOptions[] = [ { id: 0, - name: 'MIN_STK_10', + name: 'Min Carbon stock 0-10cm (kg/m²)', key: 'MIN_STK_10', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 1, - name: 'MAX_STK_10', + name: 'Max Carbon stock 0-10cm (kg/m²)', key: 'MAX_STK_10', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 2, - name: 'MIN_STK_15', + name: 'Min Carbon stock 0-15cm (kg/m²)', key: 'MIN_STK_15', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 3, - name: 'MAX_STK_15', + name: 'Max Carbon stock 0-15cm (kg/m²)', key: 'MAX_STK_15', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 4, - name: 'MIN_STK_30', + name: 'Min Carbon stock 0-30cm (kg/m²)', key: 'MIN_STK_30', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 5, - name: 'MAX_STK_30', + name: 'Max Carbon stock 0-30cm (kg/m²)', key: 'MAX_STK_30', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 6, - name: 'AV_STK_30', + name: 'Average Carbon stock 0-30cm (kg/m²)', key: 'AV_STK_30', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 7, - name: 'AV_STK_100', + name: 'Average Carbon stock 30-100cm (kg/m²)', key: 'AV_STK_100', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 8, - name: 'AV_STK_150', + name: 'Average Carbon stock 100-150cm (kg/m²)', key: 'AV_STK_150', - socket: numericDataSocket + socket: numericDataSocket, + unit: 'kg/m^2' }, { id: 9, - name: 'AV_OC_30', + name: 'Average Organic Carbon 0-30cm (%)', key: 'AV_OC_30', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' }, { id: 10, - name: 'MIN_OC_30', + name: 'Min Organic Carbon 0-30cm (%)', key: 'MIN_OC_30', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' }, { id: 11, - name: 'MAX_OC_30', + name: 'Max Organic Carbon 0-30cm (%)', key: 'MAX_OC_30', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' }, { id: 12, - name: 'AV_OC_100', + name: 'Average Organic Carbon 30-100cm (%)', key: 'AV_OC_100', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' }, { id: 13, - name: 'MIN_OC_100', + name: 'Min Organic Carbon 30-100cm (%)', key: 'MIN_OC_100', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' }, { id: 14, - name: 'MAX_OC_100', + name: 'Max Organic Carbon 30-100cm (%)', key: 'MAX_OC_100', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' }, { id: 15, - name: 'AV_OC_150', + name: 'Average Organic Carbon 100-150cm (%)', key: 'AV_OC_150', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' }, { id: 16, - name: 'MIN_OC_150', + name: 'Min Organic Carbon 100-150cm (%)', key: 'MIN_OC_150', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' }, { id: 17, - name: 'MAX_OC_150', + name: 'Max Organic Carbon 100-150cm (%)', key: 'MAX_OC_150', - socket: numericDataSocket + socket: numericDataSocket, + unit: '%' } ] diff --git a/app/javascript/projects/project_editor.tsx b/app/javascript/projects/project_editor.tsx index 188b651..bf4181d 100644 --- a/app/javascript/projects/project_editor.tsx +++ b/app/javascript/projects/project_editor.tsx @@ -25,6 +25,7 @@ export interface ProjectPermissions { DefraHedgerows: boolean KewRgb25cm: boolean KewSamples: boolean + NATMAPSoil: boolean } export interface TeamExtentData { diff --git a/app/models/project.rb b/app/models/project.rb index 1d3dd6e..369c43b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -72,6 +72,14 @@ def kew_samples_permission tp ? tp.enabled : false end + def natmap_soil_permission + p = Permission.find_by(name: 'natmap_soil') + return false unless p + + tp = team.team_permissions.find_by(permission: p) + tp ? tp.enabled : false + end + def extents team_extents = Extent.where(team_id: team.id).to_json end diff --git a/app/views/projects/show.html.erb b/app/views/projects/show.html.erb index a0a15c7..aad6ec6 100644 --- a/app/views/projects/show.html.erb +++ b/app/views/projects/show.html.erb @@ -7,6 +7,7 @@ data-projects-project-defra-hedgerow-permission-value="<%= @project.defra_hedgerow_permission %>" data-projects-project-kew-rgb25cm-permission-value="<%= @project.kew_rgb25cm_permission %>" data-projects-project-kew-samples-permission-value="<%= @project.kew_samples_permission %>" + data-projects-project-natmap-soil-permission-value="<%= @project.natmap_soil_permission %>" data-projects-project-extents-value="<%= @project.extents %>" data-projects-back-button-path-value="<%= team_projects_path(@project.team) %>" data-projects-db-models-value="<%= render partial: "defs", formats: [:json] %>" diff --git a/db/migrate/20241121163637_add_natmap_permission.rb b/db/migrate/20241121163637_add_natmap_permission.rb new file mode 100644 index 0000000..c7c07e2 --- /dev/null +++ b/db/migrate/20241121163637_add_natmap_permission.rb @@ -0,0 +1,9 @@ +class AddNatmapPermission < ActiveRecord::Migration[6.1] + def change + permission = Permission.find_or_create_by(name: 'natmap_soil') + + Team.all.each do |team| + TeamPermission.find_or_create_by(team: team, permission: permission, enabled: false) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index f5e0b7d..77248de 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_10_21_150643) do +ActiveRecord::Schema.define(version: 2024_11_21_163637) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql"