From 2704f0b8f1d94e5312175dd41bcd39493d416a82 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 14 Jan 2025 13:35:53 +0100 Subject: [PATCH] =?UTF-8?q?Refactor=20backend:=20Use=20=E2=80=9Cfolder?= =?UTF-8?q?=E2=80=9D=20only=20for=20dashboard=20folders,=20otherwise=20?= =?UTF-8?q?=E2=80=9Cdirectory=E2=80=9D=20(#8292)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refactor backend: Use “folder” only for dashboard folders, otherwise “directory” * format * migration guide pr num * adapt application.conf * adapt standalone-datastore.conf --- MIGRATIONS.unreleased.md | 1 + app/utils/WkConf.scala | 1 - conf/application.conf | 4 +- test/e2e/End2EndSpec.scala | 2 +- .../com/scalableminds/util/io/ZipIO.scala | 28 ++--- .../datastore/DataStoreConfig.scala | 4 +- .../controllers/ExportsController.scala | 2 +- .../controllers/ZarrStreamingController.scala | 112 +++++++++--------- .../explore/ExploreRemoteLayerService.scala | 4 +- .../services/AgglomerateService.scala | 2 +- .../services/BinaryDataServiceHolder.scala | 2 +- .../services/ConnectomeFileService.scala | 2 +- .../services/DSRemoteTracingstoreClient.scala | 18 +-- .../services/DSUsedStorageService.scala | 2 +- .../services/DataSourceService.scala | 2 +- .../datastore/services/MappingService.scala | 2 +- .../datastore/services/MeshFileService.scala | 2 +- .../services/SegmentIndexFileService.scala | 2 +- .../services/uploading/UploadService.scala | 2 +- .../RemoteSourceDescriptorService.scala | 4 +- ....scalableminds.webknossos.datastore.routes | 48 ++++---- .../conf/standalone-datastore.conf | 4 +- ...VolumeTracingZarrStreamingController.scala | 22 ++-- ...alableminds.webknossos.tracingstore.routes | 24 ++-- 24 files changed, 149 insertions(+), 147 deletions(-) diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md index 6e0d5ec66d8..fedd040bfda 100644 --- a/MIGRATIONS.unreleased.md +++ b/MIGRATIONS.unreleased.md @@ -9,6 +9,7 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md). [Commits](https://github.com/scalableminds/webknossos/compare/24.12.0...HEAD) - Removed support for HTTP API versions 3 and 4. [#8075](https://github.com/scalableminds/webknossos/pull/8075) - New FossilDB version `0.1.33` (docker image `scalableminds/fossildb:master__504`) is required. +- Datastore config options `datastore.baseFolder` and `localFolderWhitelist` to `datastore.baseDirectory` and `localDirectoryWhitelist` respectively, to avoid confusion with the dashboard folders. [#8292](https://github.com/scalableminds/webknossos/pull/8292) ### Postgres Evolutions: - [124-decouple-dataset-directory-from-name](conf/evolutions/124-decouple-dataset-directory-from-name) diff --git a/app/utils/WkConf.scala b/app/utils/WkConf.scala index 0272635bbf2..cd6f139f329 100644 --- a/app/utils/WkConf.scala +++ b/app/utils/WkConf.scala @@ -128,7 +128,6 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader with L val key: String = get[String]("datastore.key") val name: String = get[String]("datastore.name") val publicUri: Option[String] = getOptional[String]("datastore.publicUri") - val localFolderWhitelist: List[String] = getList[String]("datastore.localFolderWhitelist") } object Tracingstore { diff --git a/conf/application.conf b/conf/application.conf index 751fcd1e5aa..44f925475ce 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -194,8 +194,8 @@ datastore { uri = ${http.uri} pingInterval = 10 minutes } - baseFolder = "binaryData" - localFolderWhitelist = [] # list of local absolute directory paths where image data may be explored and served from + baseDirectory = "binaryData" + localDirectoryWhitelist = [] # list of local absolute directory paths where image data may be explored and served from watchFileSystem { enabled = true interval = 1 minute diff --git a/test/e2e/End2EndSpec.scala b/test/e2e/End2EndSpec.scala index dc61e6c5d38..a6572ed0f36 100644 --- a/test/e2e/End2EndSpec.scala +++ b/test/e2e/End2EndSpec.scala @@ -62,7 +62,7 @@ class End2EndSpec(arguments: Arguments) extends Specification with GuiceFakeAppl } // Skip unzipping if the test dataset is already present if (!dataDirectory.listFiles().exists(_.getName == "test-dataset")) - ZipIO.unzipToFolder( + ZipIO.unzipToDirectory( testDatasetZip, Paths.get(dataDirectory.toPath.toString, "test-dataset"), includeHiddenFiles = true, diff --git a/util/src/main/scala/com/scalableminds/util/io/ZipIO.scala b/util/src/main/scala/com/scalableminds/util/io/ZipIO.scala index d06c23b1f04..fb5a8fcf69b 100644 --- a/util/src/main/scala/com/scalableminds/util/io/ZipIO.scala +++ b/util/src/main/scala/com/scalableminds/util/io/ZipIO.scala @@ -275,21 +275,21 @@ object ZipIO extends LazyLogging { result } - def unzipToFolder(file: File, - targetDir: Path, - includeHiddenFiles: Boolean, - hiddenFilesWhitelist: List[String], - truncateCommonPrefix: Boolean, - excludeFromPrefix: Option[List[String]]): Box[List[Path]] = + def unzipToDirectory(file: File, + targetDir: Path, + includeHiddenFiles: Boolean, + hiddenFilesWhitelist: List[String], + truncateCommonPrefix: Boolean, + excludeFromPrefix: Option[List[String]]): Box[List[Path]] = tryo(new java.util.zip.ZipFile(file)).flatMap( - unzipToFolder(_, targetDir, includeHiddenFiles, hiddenFilesWhitelist, truncateCommonPrefix, excludeFromPrefix)) - - def unzipToFolder(zip: ZipFile, - targetDir: Path, - includeHiddenFiles: Boolean, - hiddenFilesWhitelist: List[String], - truncateCommonPrefix: Boolean, - excludeFromPrefix: Option[List[String]]): Box[List[Path]] = + unzipToDirectory(_, targetDir, includeHiddenFiles, hiddenFilesWhitelist, truncateCommonPrefix, excludeFromPrefix)) + + def unzipToDirectory(zip: ZipFile, + targetDir: Path, + includeHiddenFiles: Boolean, + hiddenFilesWhitelist: List[String], + truncateCommonPrefix: Boolean, + excludeFromPrefix: Option[List[String]]): Box[List[Path]] = withUnziped(zip, includeHiddenFiles, hiddenFilesWhitelist, truncateCommonPrefix, excludeFromPrefix) { (name, in) => val path = targetDir.resolve(name) if (path.getParent != null) { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala index cca12670f50..1426cf990d1 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala @@ -20,8 +20,8 @@ class DataStoreConfig @Inject()(configuration: Configuration) extends ConfigRead val uri: String = get[String]("datastore.webKnossos.uri") val pingInterval: FiniteDuration = get[FiniteDuration]("datastore.webKnossos.pingInterval") } - val baseFolder: String = get[String]("datastore.baseFolder") - val localFolderWhitelist: List[String] = getList[String]("datastore.localFolderWhitelist") + val baseDirectory: String = get[String]("datastore.baseDirectory") + val localDirectoryWhitelist: List[String] = getList[String]("datastore.localDirectoryWhitelist") object WatchFileSystem { val enabled: Boolean = get[Boolean]("datastore.watchFileSystem.enabled") val interval: FiniteDuration = get[FiniteDuration]("datastore.watchFileSystem.interval") diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala index f4777fdb4b9..7394c2d1244 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala @@ -31,7 +31,7 @@ class ExportsController @Inject()(webknossosClient: DSRemoteWebknossosClient, extends Controller with FoxImplicits { - private val dataBaseDir: Path = Paths.get(config.Datastore.baseFolder) + private val dataBaseDir: Path = Paths.get(config.Datastore.baseDirectory) override def allowRemoteOrigin: Boolean = true diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index aaeefc53e63..2c0a13f546a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -412,25 +412,25 @@ class ZarrStreamingController @Inject()( } } yield result - def requestDataLayerMagFolderContents(token: Option[String], - organizationId: String, - datasetDirectoryName: String, - dataLayerName: String, - mag: String, - zarrVersion: Int): Action[AnyContent] = + def requestDataLayerMagDirectoryContents(token: Option[String], + organizationId: String, + datasetDirectoryName: String, + dataLayerName: String, + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess( UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId)), urlOrHeaderToken(token, request)) { - dataLayerMagFolderContents(organizationId, datasetDirectoryName, dataLayerName, mag, zarrVersion) + dataLayerMagDirectoryContents(organizationId, datasetDirectoryName, dataLayerName, mag, zarrVersion) } } - private def dataLayerMagFolderContents(organizationId: String, - datasetDirectoryName: String, - dataLayerName: String, - mag: String, - zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] = + private def dataLayerMagDirectoryContents(organizationId: String, + datasetDirectoryName: String, + dataLayerName: String, + mag: String, + zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] = for { (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetDirectoryName, @@ -447,11 +447,11 @@ class ZarrStreamingController @Inject()( additionalEntries )).withHeaders() - def dataLayerMagFolderContentsPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String, - zarrVersion: Int): Action[AnyContent] = + def dataLayerMagDirectoryContentsPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( token, @@ -459,11 +459,11 @@ class ZarrStreamingController @Inject()( dataLayerName, ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => remoteTracingstoreClient - .getDataLayerMagFolderContents(annotationLayer.tracingId, - mag, - annotationSource.tracingStoreUrl, - relevantToken, - zarrVersion) + .getDataLayerMagDirectoryContents(annotationLayer.tracingId, + mag, + annotationSource.tracingStoreUrl, + relevantToken, + zarrVersion) .map( layers => Ok( @@ -473,30 +473,30 @@ class ZarrStreamingController @Inject()( layers )).withHeaders()), orElse = annotationSource => - dataLayerMagFolderContents(annotationSource.organizationId, - annotationSource.datasetDirectoryName, - dataLayerName, - mag, - zarrVersion) + dataLayerMagDirectoryContents(annotationSource.organizationId, + annotationSource.datasetDirectoryName, + dataLayerName, + mag, + zarrVersion) ) } - def requestDataLayerFolderContents(token: Option[String], - organizationId: String, - datasetDirectoryName: String, - dataLayerName: String, - zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => + def requestDataLayerDirectoryContents(token: Option[String], + organizationId: String, + datasetDirectoryName: String, + dataLayerName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess( UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId)), urlOrHeaderToken(token, request)) { - dataLayerFolderContents(organizationId, datasetDirectoryName, dataLayerName, zarrVersion) + dataLayerDirectoryContents(organizationId, datasetDirectoryName, dataLayerName, zarrVersion) } } - private def dataLayerFolderContents(organizationId: String, - datasetDirectoryName: String, - dataLayerName: String, - zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] = + private def dataLayerDirectoryContents(organizationId: String, + datasetDirectoryName: String, + dataLayerName: String, + zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] = for { (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetDirectoryName, @@ -514,10 +514,10 @@ class ZarrStreamingController @Inject()( additionalFiles ++ mags.map(_.toMagLiteral(allowScalar = true)) )).withHeaders() - def dataLayerFolderContentsPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - zarrVersion: Int): Action[AnyContent] = + def dataLayerDirectoryContentsPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( token, @@ -525,10 +525,10 @@ class ZarrStreamingController @Inject()( dataLayerName, ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => remoteTracingstoreClient - .getDataLayerFolderContents(annotationLayer.tracingId, - annotationSource.tracingStoreUrl, - relevantToken, - zarrVersion) + .getDataLayerDirectoryContents(annotationLayer.tracingId, + annotationSource.tracingStoreUrl, + relevantToken, + zarrVersion) .map( layers => Ok( @@ -538,17 +538,17 @@ class ZarrStreamingController @Inject()( layers )).withHeaders()), orElse = annotationSource => - dataLayerFolderContents(annotationSource.organizationId, - annotationSource.datasetDirectoryName, - dataLayerName, - zarrVersion) + dataLayerDirectoryContents(annotationSource.organizationId, + annotationSource.datasetDirectoryName, + dataLayerName, + zarrVersion) ) } - def requestDataSourceFolderContents(token: Option[String], - organizationId: String, - datasetDirectoryName: String, - zarrVersion: Int): Action[AnyContent] = + def requestDataSourceDirectoryContents(token: Option[String], + organizationId: String, + datasetDirectoryName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess( UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId)), @@ -570,9 +570,9 @@ class ZarrStreamingController @Inject()( } } - def dataSourceFolderContentsPrivateLink(token: Option[String], - accessToken: String, - zarrVersion: Int): Action[AnyContent] = + def dataSourceDirectoryContentsPrivateLink(token: Option[String], + accessToken: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala index 84cd905863f..3daef569eb9 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala @@ -104,8 +104,8 @@ class ExploreRemoteLayerService @Inject()(dataVaultService: DataVaultService, private def assertLocalPathInWhitelist(uri: URI)(implicit ec: ExecutionContext): Fox[Unit] = if (uri.getScheme == DataVaultService.schemeFile) { - bool2Fox(dataStoreConfig.Datastore.localFolderWhitelist.exists(whitelistEntry => - uri.getPath.startsWith(whitelistEntry))) ?~> s"Absolute path ${uri.getPath} in local file system is not in path whitelist. Consider adding it to datastore.localFolderWhitelist" + bool2Fox(dataStoreConfig.Datastore.localDirectoryWhitelist.exists(whitelistEntry => + uri.getPath.startsWith(whitelistEntry))) ?~> s"Absolute path ${uri.getPath} in local file system is not in path whitelist. Consider adding it to datastore.localDirectoryWhitelist" } else Fox.successful(()) private val MAX_RECURSIVE_SEARCH_DEPTH = 3 diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala index cd866602929..6869590c5d6 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala @@ -26,7 +26,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte private val agglomerateDir = "agglomerates" private val agglomerateFileExtension = "hdf5" private val datasetName = "/segment_to_agglomerate" - private val dataBaseDir = Paths.get(config.Datastore.baseFolder) + private val dataBaseDir = Paths.get(config.Datastore.baseDirectory) private val cumsumFileName = "cumsum.json" lazy val agglomerateFileCache = new AgglomerateFileCache(config.Datastore.Cache.AgglomerateFile.maxFileHandleEntries) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala index 9bca015857b..09c34a3bc08 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala @@ -42,7 +42,7 @@ class BinaryDataServiceHolder @Inject()( } val binaryDataService: BinaryDataService = new BinaryDataService( - Paths.get(config.Datastore.baseFolder), + Paths.get(config.Datastore.baseDirectory), Some(agglomerateService), Some(remoteSourceDescriptorService), Some(sharedChunkContentsCache), diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/ConnectomeFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/ConnectomeFileService.scala index fc9477df11e..14269bcb60a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/ConnectomeFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/ConnectomeFileService.scala @@ -84,7 +84,7 @@ class ConnectomeFileService @Inject()(config: DataStoreConfig)(implicit ec: Exec extends FoxImplicits with LazyLogging { - private val dataBaseDir = Paths.get(config.Datastore.baseFolder) + private val dataBaseDir = Paths.get(config.Datastore.baseDirectory) private val connectomesDir = "connectomes" private val connectomeFileExtension = "hdf5" diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 2924c0687e4..25849a910a6 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -74,19 +74,19 @@ class DSRemoteTracingstoreClient @Inject()( .addQueryStringOptional("token", token) .getWithBytesResponse - def getDataLayerMagFolderContents(tracingId: String, - mag: String, - tracingStoreUri: String, - token: Option[String], - zarrVersion: Int): Fox[List[String]] = + def getDataLayerMagDirectoryContents(tracingId: String, + mag: String, + tracingStoreUri: String, + token: Option[String], + zarrVersion: Int): Fox[List[String]] = rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId/$mag") .addQueryStringOptional("token", token) .getWithJsonResponse[List[String]] - def getDataLayerFolderContents(tracingId: String, - tracingStoreUri: String, - token: Option[String], - zarrVersion: Int): Fox[List[String]] = + def getDataLayerDirectoryContents(tracingId: String, + tracingStoreUri: String, + token: Option[String], + zarrVersion: Int): Fox[List[String]] = rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId") .addQueryStringOptional("token", token) .getWithJsonResponse[List[String]] diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala index d35ea66b840..6a0819b8c7c 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala @@ -29,7 +29,7 @@ class DSUsedStorageService @Inject()(config: DataStoreConfig)(implicit ec: Execu extends FoxImplicits with LazyLogging { - private val baseDir: Path = Paths.get(config.Datastore.baseFolder) + private val baseDir: Path = Paths.get(config.Datastore.baseDirectory) private def noSymlinksFilter(p: Path) = !Files.isSymbolicLink(p) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala index 1e5cd8fcf40..c7ab44f71f0 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala @@ -41,7 +41,7 @@ class DataSourceService @Inject()( override protected def tickerInitialDelay: FiniteDuration = config.Datastore.WatchFileSystem.initialDelay - val dataBaseDir: Path = Paths.get(config.Datastore.baseFolder) + val dataBaseDir: Path = Paths.get(config.Datastore.baseDirectory) private val propertiesFileName = Paths.get(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON) private val logFileName = Paths.get("datasource-properties-backups.log") diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MappingService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MappingService.scala index 3da55a2efcd..f325ad5f227 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MappingService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MappingService.scala @@ -20,7 +20,7 @@ class MappingService @Inject()(config: DataStoreConfig)(implicit ec: ExecutionCo def handleMappingRequest(request: DataServiceMappingRequest): Fox[Array[Byte]] = { val readInstruction = - MappingReadInstruction(Paths.get(config.Datastore.baseFolder), request.dataSource, request.mapping) + MappingReadInstruction(Paths.get(config.Datastore.baseDirectory), request.dataSource, request.mapping) request.dataLayer.mappingProvider.load(readInstruction) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshFileService.scala index eaf88ad19c0..41916d5a6b8 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshFileService.scala @@ -177,7 +177,7 @@ class MeshFileService @Inject()(config: DataStoreConfig)(implicit ec: ExecutionC with Hdf5HashedArrayUtils with ByteUtils { - private val dataBaseDir = Paths.get(config.Datastore.baseFolder) + private val dataBaseDir = Paths.get(config.Datastore.baseDirectory) private val meshesDir = "meshes" private lazy val meshFileCache = new Hdf5FileCache(30) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala index 02c907935d3..23e7b2eb467 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala @@ -34,7 +34,7 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig, extends FoxImplicits with Hdf5HashedArrayUtils with SegmentStatistics { - private val dataBaseDir = Paths.get(config.Datastore.baseFolder) + private val dataBaseDir = Paths.get(config.Datastore.baseDirectory) private val segmentIndexDir = "segmentIndex" private lazy val fileHandleCache = new Hdf5FileCache(10) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala index ec262c77e7a..98898719f33 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala @@ -589,7 +589,7 @@ class UploadService @Inject()(dataSourceRepository: DataSourceRepository, _ <- if (shallowFileList.length == 1 && shallowFileList.headOption.exists( _.toString.toLowerCase.endsWith(".zip"))) { firstFile.toFox.flatMap { file => - ZipIO.unzipToFolder( + ZipIO.unzipToDirectory( new File(file.toString), unpackToDir, includeHiddenFiles = false, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RemoteSourceDescriptorService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RemoteSourceDescriptorService.scala index a33177eb321..a109c5c8e0f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RemoteSourceDescriptorService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RemoteSourceDescriptorService.scala @@ -60,12 +60,12 @@ class RemoteSourceDescriptorService @Inject()(dSRemoteWebknossosClient: DSRemote } else if (uri.getScheme == null || uri.getScheme == DataVaultService.schemeFile) { val localPath = Paths.get(uri.getPath) if (localPath.isAbsolute) { - if (dataStoreConfig.Datastore.localFolderWhitelist.exists(whitelistEntry => + if (dataStoreConfig.Datastore.localDirectoryWhitelist.exists(whitelistEntry => localPath.toString.startsWith(whitelistEntry))) uri else throw new Exception( - s"Absolute path $localPath in local file system is not in path whitelist. Consider adding it to datastore.localFolderWhitelist") + s"Absolute path $localPath in local file system is not in path whitelist. Consider adding it to datastore.localDirectoryWhitelist") } else { // relative local path, resolve in dataset dir val magPathRelativeToDataset = localDatasetDir.resolve(localPath) val magPathRelativeToLayer = localDatasetDir.resolve(layerName).resolve(localPath) diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index 170ed69c5b0..8ea8c79095d 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -16,52 +16,52 @@ GET /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerN GET /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/mag:mag/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: Int, x: Int, y: Int, z: Int, cubeSize: Int) # Zarr2 compatible routes -GET /zarr/:organizationId/:datasetDirectoryName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetDirectoryName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetDirectoryName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetDirectoryName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 2) GET /zarr/:organizationId/:datasetDirectoryName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName="") GET /zarr/:organizationId/:datasetDirectoryName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, zarrVersion: Int = 2) GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZAttrs(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String) GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String) -GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZArray(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String) GET /zarr/:organizationId/:datasetDirectoryName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, coordinates: String) -GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName="") GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zAttrsWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArrayPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) # Zarr3 compatible routes -GET /zarr3_experimental/:organizationId/:datasetDirectoryName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetDirectoryName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetDirectoryName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetDirectoryName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 3) GET /zarr3_experimental/:organizationId/:datasetDirectoryName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationId: String, datasetDirectoryName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, zarrVersion: Int = 3) GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJson(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String) -GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String) GET /zarr3_experimental/:organizationId/:datasetDirectoryName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: String, coordinates: String) -GET /annotations/zarr3_experimental/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) GET /annotations/zarr3_experimental/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) diff --git a/webknossos-datastore/conf/standalone-datastore.conf b/webknossos-datastore/conf/standalone-datastore.conf index 1b035ea09c0..78c12f3e04c 100644 --- a/webknossos-datastore/conf/standalone-datastore.conf +++ b/webknossos-datastore/conf/standalone-datastore.conf @@ -54,8 +54,8 @@ datastore { uri = "http://localhost:9000" pingInterval = 10 minutes } - baseFolder = "binaryData" - localFolderWhitelist = [] # list of local absolute directory paths where image data may be explored and served from + baseDirectory = "binaryData" + localDirectoryWhitelist = [] # list of local absolute directory paths where image data may be explored and served from watchFileSystem { enabled = true interval = 1 minute diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 64094d38910..07a0edad91d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -56,7 +56,7 @@ class VolumeTracingZarrStreamingController @Inject()( override def defaultErrorCode: Int = NOT_FOUND - def volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = + def volumeTracingDirectoryContent(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -75,7 +75,9 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = + def volumeTracingDirectoryContentJson(token: Option[String], + tracingId: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -88,10 +90,10 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingMagFolderContent(token: Option[String], - tracingId: String, - mag: String, - zarrVersion: Int): Action[AnyContent] = + def volumeTracingMagDirectoryContent(token: Option[String], + tracingId: String, + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -111,10 +113,10 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingMagFolderContentJson(token: Option[String], - tracingId: String, - mag: String, - zarrVersion: Int): Action[AnyContent] = + def volumeTracingMagDirectoryContentJson(token: Option[String], + tracingId: String, + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index d1384d8aa4b..f04a4ea65c5 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -41,27 +41,27 @@ POST /mapping/:tracingId/agglomeratesForSegments @com.scalablemin # Zarr endpoints for volume annotations # Zarr version 2 -GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingDirectoryContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagDirectoryContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingDirectoryContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingDirectoryContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) GET /volume/zarr/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(token: Option[String], tracingId: String) GET /volume/zarr/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(token: Option[String], tracingId: String) GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) -GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagDirectoryContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagDirectoryContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) GET /volume/zarr/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) # Zarr version 3 -GET /volume/zarr3_experimental/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingDirectoryContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagDirectoryContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingDirectoryContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingDirectoryContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) GET /volume/zarr3_experimental/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) GET /volume/zarr3_experimental/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], tracingId: String) -GET /volume/zarr3_experimental/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagDirectoryContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagDirectoryContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], tracingId: String, mag: String) GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String)