Refactor backend: Use “folder” only for dashboard folders, otherwise …

…“directory” (#8292)

* Refactor backend: Use “folder” only for dashboard folders, otherwise “directory”

* format

* migration guide pr num

* adapt application.conf

* adapt standalone-datastore.conf

MIGRATIONS.unreleased.md

@@ -9,6 +9,7 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 [Commits](https://github.com/scalableminds/webknossos/compare/24.12.0...HEAD)
 - Removed support for HTTP API versions 3 and 4. [#8075](https://github.com/scalableminds/webknossos/pull/8075)
 - New FossilDB version `0.1.33` (docker image `scalableminds/fossildb:master__504`) is required.
+- Datastore config options `datastore.baseFolder` and `localFolderWhitelist` to `datastore.baseDirectory` and `localDirectoryWhitelist` respectively, to avoid confusion with the dashboard folders. [#8292](https://github.com/scalableminds/webknossos/pull/8292)
 
 ### Postgres Evolutions:
 - [124-decouple-dataset-directory-from-name](conf/evolutions/124-decouple-dataset-directory-from-name)

app/utils/WkConf.scala

@@ -128,7 +128,6 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader with L
     val key: String = get[String]("datastore.key")
     val name: String = get[String]("datastore.name")
     val publicUri: Option[String] = getOptional[String]("datastore.publicUri")
-    val localFolderWhitelist: List[String] = getList[String]("datastore.localFolderWhitelist")
   }
 
   object Tracingstore {

conf/application.conf

@@ -194,8 +194,8 @@ datastore {
     uri = ${http.uri}
     pingInterval = 10 minutes
   }
-  baseFolder = "binaryData"
-  localFolderWhitelist = [] # list of local absolute directory paths where image data may be explored and served from
+  baseDirectory = "binaryData"
+  localDirectoryWhitelist = [] # list of local absolute directory paths where image data may be explored and served from
   watchFileSystem {
     enabled = true
     interval = 1 minute

test/e2e/End2EndSpec.scala

@@ -62,7 +62,7 @@ class End2EndSpec(arguments: Arguments) extends Specification with GuiceFakeAppl
     }
     // Skip unzipping if the test dataset is already present
     if (!dataDirectory.listFiles().exists(_.getName == "test-dataset"))
-      ZipIO.unzipToFolder(
+      ZipIO.unzipToDirectory(
         testDatasetZip,
         Paths.get(dataDirectory.toPath.toString, "test-dataset"),
         includeHiddenFiles = true,

util/src/main/scala/com/scalableminds/util/io/ZipIO.scala

@@ -275,21 +275,21 @@ object ZipIO extends LazyLogging {
     result
   }
 
-  def unzipToFolder(file: File,
-                    targetDir: Path,
-                    includeHiddenFiles: Boolean,
-                    hiddenFilesWhitelist: List[String],
-                    truncateCommonPrefix: Boolean,
-                    excludeFromPrefix: Option[List[String]]): Box[List[Path]] =
+  def unzipToDirectory(file: File,
+                       targetDir: Path,
+                       includeHiddenFiles: Boolean,
+                       hiddenFilesWhitelist: List[String],
+                       truncateCommonPrefix: Boolean,
+                       excludeFromPrefix: Option[List[String]]): Box[List[Path]] =
     tryo(new java.util.zip.ZipFile(file)).flatMap(
-      unzipToFolder(_, targetDir, includeHiddenFiles, hiddenFilesWhitelist, truncateCommonPrefix, excludeFromPrefix))
-
-  def unzipToFolder(zip: ZipFile,
-                    targetDir: Path,
-                    includeHiddenFiles: Boolean,
-                    hiddenFilesWhitelist: List[String],
-                    truncateCommonPrefix: Boolean,
-                    excludeFromPrefix: Option[List[String]]): Box[List[Path]] =
+      unzipToDirectory(_, targetDir, includeHiddenFiles, hiddenFilesWhitelist, truncateCommonPrefix, excludeFromPrefix))
+
+  def unzipToDirectory(zip: ZipFile,
+                       targetDir: Path,
+                       includeHiddenFiles: Boolean,
+                       hiddenFilesWhitelist: List[String],
+                       truncateCommonPrefix: Boolean,
+                       excludeFromPrefix: Option[List[String]]): Box[List[Path]] =
     withUnziped(zip, includeHiddenFiles, hiddenFilesWhitelist, truncateCommonPrefix, excludeFromPrefix) { (name, in) =>
       val path = targetDir.resolve(name)
       if (path.getParent != null) {

webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala

@@ -20,8 +20,8 @@ class DataStoreConfig @Inject()(configuration: Configuration) extends ConfigRead
       val uri: String = get[String]("datastore.webKnossos.uri")
       val pingInterval: FiniteDuration = get[FiniteDuration]("datastore.webKnossos.pingInterval")
     }
-    val baseFolder: String = get[String]("datastore.baseFolder")
-    val localFolderWhitelist: List[String] = getList[String]("datastore.localFolderWhitelist")
+    val baseDirectory: String = get[String]("datastore.baseDirectory")
+    val localDirectoryWhitelist: List[String] = getList[String]("datastore.localDirectoryWhitelist")
     object WatchFileSystem {
       val enabled: Boolean = get[Boolean]("datastore.watchFileSystem.enabled")
       val interval: FiniteDuration = get[FiniteDuration]("datastore.watchFileSystem.interval")

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala

@@ -31,7 +31,7 @@ class ExportsController @Inject()(webknossosClient: DSRemoteWebknossosClient,
     extends Controller
     with FoxImplicits {
 
-  private val dataBaseDir: Path = Paths.get(config.Datastore.baseFolder)
+  private val dataBaseDir: Path = Paths.get(config.Datastore.baseDirectory)
 
   override def allowRemoteOrigin: Boolean = true
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala

@@ -412,25 +412,25 @@ class ZarrStreamingController @Inject()(
       }
     } yield result
 
-  def requestDataLayerMagFolderContents(token: Option[String],
-                                        organizationId: String,
-                                        datasetDirectoryName: String,
-                                        dataLayerName: String,
-                                        mag: String,
-                                        zarrVersion: Int): Action[AnyContent] =
+  def requestDataLayerMagDirectoryContents(token: Option[String],
+                                           organizationId: String,
+                                           datasetDirectoryName: String,
+                                           dataLayerName: String,
+                                           mag: String,
+                                           zarrVersion: Int): Action[AnyContent] =
     Action.async { implicit request =>
       accessTokenService.validateAccess(
         UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId)),
         urlOrHeaderToken(token, request)) {
-        dataLayerMagFolderContents(organizationId, datasetDirectoryName, dataLayerName, mag, zarrVersion)
+        dataLayerMagDirectoryContents(organizationId, datasetDirectoryName, dataLayerName, mag, zarrVersion)
       }
     }
 
-  private def dataLayerMagFolderContents(organizationId: String,
-                                         datasetDirectoryName: String,
-                                         dataLayerName: String,
-                                         mag: String,
-                                         zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] =
+  private def dataLayerMagDirectoryContents(organizationId: String,
+                                            datasetDirectoryName: String,
+                                            dataLayerName: String,
+                                            mag: String,
+                                            zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] =
     for {
       (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
                                                                        datasetDirectoryName,
@@ -447,23 +447,23 @@ class ZarrStreamingController @Inject()(
           additionalEntries
         )).withHeaders()
 
-  def dataLayerMagFolderContentsPrivateLink(token: Option[String],
-                                            accessToken: String,
-                                            dataLayerName: String,
-                                            mag: String,
-                                            zarrVersion: Int): Action[AnyContent] =
+  def dataLayerMagDirectoryContentsPrivateLink(token: Option[String],
+                                               accessToken: String,
+                                               dataLayerName: String,
+                                               mag: String,
+                                               zarrVersion: Int): Action[AnyContent] =
     Action.async { implicit request =>
       ifIsAnnotationLayerOrElse(
         token,
         accessToken,
         dataLayerName,
         ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) =>
           remoteTracingstoreClient
-            .getDataLayerMagFolderContents(annotationLayer.tracingId,
-                                           mag,
-                                           annotationSource.tracingStoreUrl,
-                                           relevantToken,
-                                           zarrVersion)
+            .getDataLayerMagDirectoryContents(annotationLayer.tracingId,
+                                              mag,
+                                              annotationSource.tracingStoreUrl,
+                                              relevantToken,
+                                              zarrVersion)
             .map(
               layers =>
                 Ok(
@@ -473,30 +473,30 @@ class ZarrStreamingController @Inject()(
                     layers
                   )).withHeaders()),
         orElse = annotationSource =>
-          dataLayerMagFolderContents(annotationSource.organizationId,
-                                     annotationSource.datasetDirectoryName,
-                                     dataLayerName,
-                                     mag,
-                                     zarrVersion)
+          dataLayerMagDirectoryContents(annotationSource.organizationId,
+                                        annotationSource.datasetDirectoryName,
+                                        dataLayerName,
+                                        mag,
+                                        zarrVersion)
       )
     }
 
-  def requestDataLayerFolderContents(token: Option[String],
-                                     organizationId: String,
-                                     datasetDirectoryName: String,
-                                     dataLayerName: String,
-                                     zarrVersion: Int): Action[AnyContent] = Action.async { implicit request =>
+  def requestDataLayerDirectoryContents(token: Option[String],
+                                        organizationId: String,
+                                        datasetDirectoryName: String,
+                                        dataLayerName: String,
+                                        zarrVersion: Int): Action[AnyContent] = Action.async { implicit request =>
     accessTokenService.validateAccess(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId)),
       urlOrHeaderToken(token, request)) {
-      dataLayerFolderContents(organizationId, datasetDirectoryName, dataLayerName, zarrVersion)
+      dataLayerDirectoryContents(organizationId, datasetDirectoryName, dataLayerName, zarrVersion)
     }
   }
 
-  private def dataLayerFolderContents(organizationId: String,
-                                      datasetDirectoryName: String,
-                                      dataLayerName: String,
-                                      zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] =
+  private def dataLayerDirectoryContents(organizationId: String,
+                                         datasetDirectoryName: String,
+                                         dataLayerName: String,
+                                         zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] =
     for {
       (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
                                                                        datasetDirectoryName,
@@ -514,21 +514,21 @@ class ZarrStreamingController @Inject()(
           additionalFiles ++ mags.map(_.toMagLiteral(allowScalar = true))
         )).withHeaders()
 
-  def dataLayerFolderContentsPrivateLink(token: Option[String],
-                                         accessToken: String,
-                                         dataLayerName: String,
-                                         zarrVersion: Int): Action[AnyContent] =
+  def dataLayerDirectoryContentsPrivateLink(token: Option[String],
+                                            accessToken: String,
+                                            dataLayerName: String,
+                                            zarrVersion: Int): Action[AnyContent] =
     Action.async { implicit request =>
       ifIsAnnotationLayerOrElse(
         token,
         accessToken,
         dataLayerName,
         ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) =>
           remoteTracingstoreClient
-            .getDataLayerFolderContents(annotationLayer.tracingId,
-                                        annotationSource.tracingStoreUrl,
-                                        relevantToken,
-                                        zarrVersion)
+            .getDataLayerDirectoryContents(annotationLayer.tracingId,
+                                           annotationSource.tracingStoreUrl,
+                                           relevantToken,
+                                           zarrVersion)
             .map(
               layers =>
                 Ok(
@@ -538,17 +538,17 @@ class ZarrStreamingController @Inject()(
                     layers
                   )).withHeaders()),
         orElse = annotationSource =>
-          dataLayerFolderContents(annotationSource.organizationId,
-                                  annotationSource.datasetDirectoryName,
-                                  dataLayerName,
-                                  zarrVersion)
+          dataLayerDirectoryContents(annotationSource.organizationId,
+                                     annotationSource.datasetDirectoryName,
+                                     dataLayerName,
+                                     zarrVersion)
       )
     }
 
-  def requestDataSourceFolderContents(token: Option[String],
-                                      organizationId: String,
-                                      datasetDirectoryName: String,
-                                      zarrVersion: Int): Action[AnyContent] =
+  def requestDataSourceDirectoryContents(token: Option[String],
+                                         organizationId: String,
+                                         datasetDirectoryName: String,
+                                         zarrVersion: Int): Action[AnyContent] =
     Action.async { implicit request =>
       accessTokenService.validateAccess(
         UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId)),
@@ -570,9 +570,9 @@ class ZarrStreamingController @Inject()(
       }
     }
 
-  def dataSourceFolderContentsPrivateLink(token: Option[String],
-                                          accessToken: String,
-                                          zarrVersion: Int): Action[AnyContent] =
+  def dataSourceDirectoryContentsPrivateLink(token: Option[String],
+                                             accessToken: String,
+                                             zarrVersion: Int): Action[AnyContent] =
     Action.async { implicit request =>
       for {
         annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request))

webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala

@@ -104,8 +104,8 @@ class ExploreRemoteLayerService @Inject()(dataVaultService: DataVaultService,
 
   private def assertLocalPathInWhitelist(uri: URI)(implicit ec: ExecutionContext): Fox[Unit] =
     if (uri.getScheme == DataVaultService.schemeFile) {
-      bool2Fox(dataStoreConfig.Datastore.localFolderWhitelist.exists(whitelistEntry =>
-        uri.getPath.startsWith(whitelistEntry))) ?~> s"Absolute path ${uri.getPath} in local file system is not in path whitelist. Consider adding it to datastore.localFolderWhitelist"
+      bool2Fox(dataStoreConfig.Datastore.localDirectoryWhitelist.exists(whitelistEntry =>
+        uri.getPath.startsWith(whitelistEntry))) ?~> s"Absolute path ${uri.getPath} in local file system is not in path whitelist. Consider adding it to datastore.localDirectoryWhitelist"
     } else Fox.successful(())
 
   private val MAX_RECURSIVE_SEARCH_DEPTH = 3

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala

@@ -26,7 +26,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
   private val agglomerateDir = "agglomerates"
   private val agglomerateFileExtension = "hdf5"
   private val datasetName = "/segment_to_agglomerate"
-  private val dataBaseDir = Paths.get(config.Datastore.baseFolder)
+  private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
   private val cumsumFileName = "cumsum.json"
 
   lazy val agglomerateFileCache = new AgglomerateFileCache(config.Datastore.Cache.AgglomerateFile.maxFileHandleEntries)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala

@@ -42,7 +42,7 @@ class BinaryDataServiceHolder @Inject()(
   }
 
   val binaryDataService: BinaryDataService = new BinaryDataService(
-    Paths.get(config.Datastore.baseFolder),
+    Paths.get(config.Datastore.baseDirectory),
     Some(agglomerateService),
     Some(remoteSourceDescriptorService),
     Some(sharedChunkContentsCache),

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/ConnectomeFileService.scala

@@ -84,7 +84,7 @@ class ConnectomeFileService @Inject()(config: DataStoreConfig)(implicit ec: Exec
     extends FoxImplicits
     with LazyLogging {
 
-  private val dataBaseDir = Paths.get(config.Datastore.baseFolder)
+  private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
   private val connectomesDir = "connectomes"
   private val connectomeFileExtension = "hdf5"
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala

@@ -74,19 +74,19 @@ class DSRemoteTracingstoreClient @Inject()(
       .addQueryStringOptional("token", token)
       .getWithBytesResponse
 
-  def getDataLayerMagFolderContents(tracingId: String,
-                                    mag: String,
-                                    tracingStoreUri: String,
-                                    token: Option[String],
-                                    zarrVersion: Int): Fox[List[String]] =
+  def getDataLayerMagDirectoryContents(tracingId: String,
+                                       mag: String,
+                                       tracingStoreUri: String,
+                                       token: Option[String],
+                                       zarrVersion: Int): Fox[List[String]] =
     rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId/$mag")
       .addQueryStringOptional("token", token)
       .getWithJsonResponse[List[String]]
 
-  def getDataLayerFolderContents(tracingId: String,
-                                 tracingStoreUri: String,
-                                 token: Option[String],
-                                 zarrVersion: Int): Fox[List[String]] =
+  def getDataLayerDirectoryContents(tracingId: String,
+                                    tracingStoreUri: String,
+                                    token: Option[String],
+                                    zarrVersion: Int): Fox[List[String]] =
     rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId")
       .addQueryStringOptional("token", token)
       .getWithJsonResponse[List[String]]

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala

@@ -29,7 +29,7 @@ class DSUsedStorageService @Inject()(config: DataStoreConfig)(implicit ec: Execu
     extends FoxImplicits
     with LazyLogging {
 
-  private val baseDir: Path = Paths.get(config.Datastore.baseFolder)
+  private val baseDir: Path = Paths.get(config.Datastore.baseDirectory)
 
   private def noSymlinksFilter(p: Path) = !Files.isSymbolicLink(p)
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala

@@ -41,7 +41,7 @@ class DataSourceService @Inject()(
 
   override protected def tickerInitialDelay: FiniteDuration = config.Datastore.WatchFileSystem.initialDelay
 
-  val dataBaseDir: Path = Paths.get(config.Datastore.baseFolder)
+  val dataBaseDir: Path = Paths.get(config.Datastore.baseDirectory)
 
   private val propertiesFileName = Paths.get(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON)
   private val logFileName = Paths.get("datasource-properties-backups.log")