Remove Datastore config reportUsedStorage.enabled. Set in DB instead

app/controllers/WKRemoteDataStoreController.scala

@@ -157,11 +157,9 @@ class WKRemoteDataStoreController @Inject()(
     implicit request =>
       dataStoreService.validateAccess(name, key) { _ =>
         val okLabel = if (request.body.ok) "ok" else "not ok"
-        logger.debug(s"Status update from data store '$name'. Status $okLabel")
+        logger.debug(s"Status update from data store ‘$name’. Status $okLabel")
         for {
           _ <- dataStoreDAO.updateUrlByName(name, request.body.url)
-          _ <- dataStoreDAO.updateReportUsedStorageEnabledByName(name,
-                                                                 request.body.reportUsedStorageEnabled.getOrElse(false))
         } yield Ok
       }
   }

app/models/dataset/DataStore.scala

@@ -163,12 +163,6 @@ class DataStoreDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext
     for { _ <- run(query.update(url)) } yield ()
   }
 
-  def updateReportUsedStorageEnabledByName(name: String, reportUsedStorageEnabled: Boolean): Fox[Unit] =
-    for {
-      _ <- run(
-        q"UPDATE webknossos.dataStores SET reportUsedStorageEnabled = $reportUsedStorageEnabled WHERE name = $name".asUpdate)
-    } yield ()
-
   def insertOne(d: DataStore): Fox[Unit] =
     for {
       _ <- run(q"""INSERT INTO webknossos.dataStores

conf/application.conf

@@ -96,6 +96,7 @@ webKnossos {
   }
   annotation.mutex.expiryTime = 2 minutes
   fetchUsedStorage {
+    # Note that storage is only scanned for datastores that have `reportUsedStorageEnabled=TRUE` in postgres.
     rescanInterval = 24 hours # do not scan organizations whose last scan is more recent than this
     tickerInterval = 10 minutes # scan some organizations at each tick
     scansPerTick = 10 # scan x organizations at each tick
@@ -220,7 +221,6 @@ datastore {
     interval = 1 minute
     initialDelay = 5 seconds
   }
-  reportUsedStorage.enabled = false
   cache {
     mapping.maxEntries = 5
     imageArrayChunks.maxSizeBytes = 2000000000 # 2 GB

test/db/dataStores.csv

@@ -1,2 +1,2 @@
 name,url,publicUrl,key,isScratch,isDeleted,allowsUpload,allowsManualUpload,onlyAllowedOrganization,reportUsedStorageEnabled
-'localhost','http://localhost:9000','http://localhost:9000','something-secure',f,f,t,t,,f
+'localhost','http://localhost:9000','http://localhost:9000','something-secure',f,f,t,t,,t

unreleased_changes/9024.md

@@ -0,0 +1,5 @@
+### Changed
+- The datastore config option `reportUsedStorage.enabled` was removed. To enable storage scan, set it in postgres directly.
+
+### Migration
+- The datastore config option `reportUsedStorage.enabled` was removed. To enable storage scan for a datastore, set reportUsedStorageEnabled to TRUE directly in the postgres table `webknossos.dataStores`.

webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala

@@ -42,7 +42,7 @@ class DataStoreConfig @Inject()(configuration: Configuration) extends ConfigRead
         val blockSize: Int = get[Int]("datastore.cache.agglomerateFile.blockSize")
         val cumsumMaxReaderRange: Long = get[Long]("datastore.cache.agglomerateFile.cumsumMaxReaderRange")
       }
-      val children = List(Mapping, AgglomerateFile)
+      val children = List(Mapping, ImageArrayChunks, AgglomerateFile)
     }
     object AdHocMesh {
       val timeout: FiniteDuration = get[FiniteDuration]("datastore.adHocMesh.timeout")
@@ -55,9 +55,6 @@ class DataStoreConfig @Inject()(configuration: Configuration) extends ConfigRead
     object AgglomerateSkeleton {
       val maxEdges: Int = get[Int]("datastore.agglomerateSkeleton.maxEdges")
     }
-    object ReportUsedStorage {
-      val enabled: Boolean = get[Boolean]("datastore.reportUsedStorage.enabled")
-    }
     object DataVaults {
       val credentials: List[Config] = getList[Config]("datastore.dataVaults.credentials")
     }
@@ -66,7 +63,7 @@ class DataStoreConfig @Inject()(configuration: Configuration) extends ConfigRead
       val objectKeyPrefix: String = get[String]("datastore.s3Upload.objectKeyPrefix")
       val credentialName: String = get[String]("datastore.s3Upload.credentialName")
     }
-    val children = List(WebKnossos, WatchFileSystem, Cache, AdHocMesh, Redis, AgglomerateSkeleton, S3Upload)
+    val children = List(WebKnossos, WatchFileSystem, Cache, AdHocMesh, Redis, AgglomerateSkeleton, DataVaults, S3Upload)
   }
 
   object SlackNotifications {

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala

@@ -28,7 +28,7 @@ import play.api.libs.json.{Json, OFormat}
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 
-case class DataStoreStatus(ok: Boolean, url: String, reportUsedStorageEnabled: Option[Boolean] = None)
+case class DataStoreStatus(ok: Boolean, url: String)
 object DataStoreStatus {
   implicit val jsonFormat: OFormat[DataStoreStatus] = Json.format[DataStoreStatus]
 }
@@ -68,7 +68,6 @@ class DSRemoteWebknossosClient @Inject()(
   private val dataStoreKey: String = config.Datastore.key
   private val dataStoreName: String = config.Datastore.name
   private val dataStoreUri: String = config.Http.uri
-  private val reportUsedStorageEnabled: Boolean = config.Datastore.ReportUsedStorage.enabled
 
   private val webknossosUri: String = config.Datastore.WebKnossos.uri
 
@@ -79,7 +78,7 @@ class DSRemoteWebknossosClient @Inject()(
   private def reportStatus(): Fox[_] =
     rpc(s"$webknossosUri/api/datastores/$dataStoreName/status")
       .addQueryParam("key", dataStoreKey)
-      .patchJson(DataStoreStatus(ok = true, dataStoreUri, Some(reportUsedStorageEnabled)))
+      .patchJson(DataStoreStatus(ok = true, dataStoreUri))
 
   def reportDataSource(dataSource: DataSource): Fox[_] =
     rpc(s"$webknossosUri/api/datastores/$dataStoreName/datasource")

webknossos-datastore/conf/standalone-datastore.conf

@@ -80,7 +80,6 @@ datastore {
     port = 6379
   }
   agglomerateSkeleton.maxEdges = 10000
-  reportUsedStorage.enabled = false
   dataVaults {
     # Global data vault access credentials. Not available during explore, only during data loading.
     # Example: {type = "S3AccessKey", name = "s3://example/uri/prefix", identifier = "someKeyId", secret = "someSecret"}