Move dirty from Context to EagerDir

skiplang/prelude/src/skstore/Context.sk

@@ -324,7 +324,7 @@ mutable class Context private {
   mutable newDirs: SortedSet<DirName> = SortedSet[],
   mutable globals: SortedMap<String, File> = SortedMap[],
   private mutable persistents: SortedMap<String, File> = SortedMap[],
-  private mutable dirty: SortedMap<DirName, SortedSet<Key>> = SortedMap[],
+  private mutable dirsWithDirty: SortedSet<DirName> = SortedSet[],
   private mutable dirtyReaders: /* parentName => childName => keys */ SortedMap<
     DirName,
     SortedMap<DirName, SortedSet<Key>>,
@@ -435,7 +435,7 @@ mutable class Context private {
       newDirs => this.newDirs,
       globals => this.globals,
       persistents => this.persistents,
-      dirty => this.dirty,
+      dirsWithDirty => this.dirsWithDirty,
       dirtyReaders => this.dirtyReaders,
       canReuse => this.canReuse,
       arrowStack => this.arrowStack,
@@ -466,7 +466,7 @@ mutable class Context private {
     this.!newDirs = ctx.newDirs;
     this.!globals = ctx.globals;
     this.!persistents = ctx.persistents;
-    this.!dirty = ctx.dirty;
+    this.!dirsWithDirty = ctx.dirsWithDirty;
     this.!dirtyReaders = ctx.dirtyReaders;
     this.!canReuse = ctx.canReuse;
     this.!arrowStack = ctx.arrowStack;
@@ -583,7 +583,7 @@ mutable class Context private {
    * recomputed.
    *
    * - dirtyReaders are files that are made dirty because of a read access
-   * - dirty are files that are made dirty because of a Map.
+   * - dirsWithDirty are dirs containing files that are made dirty because of a Map.
    */
   /*****************************************************************************/
   mutable fun updateDirtyReaders(path: Path): void {
@@ -666,11 +666,8 @@ mutable class Context private {
     this.!lazyGets = this.lazyGets.set(Path::create(dirName, key))
   }
 
-  mutable fun addDirty(dirName: DirName, key: Key): void {
-    this.!dirty[dirName] = this.dirty.maybeGet(dirName) match {
-    | None() -> SortedSet[key]
-    | Some(set) -> set.set(key)
-    }
+  mutable fun addDirWithDirty(dirName: DirName): void {
+    this.!dirsWithDirty = this.dirsWithDirty.set(dirName);
   }
 
   mutable fun update(): void {
@@ -694,7 +691,13 @@ mutable class Context private {
     loop {
       this.toUpdate.removeMin() match {
       | None() ->
-        this.!dirty = SortedMap[];
+        for (dirName in this.dirsWithDirty) {
+          this.unsafeMaybeGetEagerDir(dirName) match {
+          | None() -> void
+          | Some(dir) -> this.setDir(dir.resetDirty())
+          }
+        };
+        this.!dirsWithDirty = SortedSet[];
         this.!dirtyReaders = SortedMap[];
         withPostponables = this.checkPostponables();
         withPre = this.updatePre();
@@ -717,7 +720,6 @@ mutable class Context private {
           };
           EagerDir::update(
             this,
-            this.dirty.maybeGet(parentName),
             this.dirtyReaders.maybeGet(parentName),
             parentName,
             parentMaps,

skiplang/prelude/src/skstore/EagerDir.sk

@@ -636,6 +636,8 @@ class EagerDir protected {
   childDirs: SortedSet<DirName> = SortedSet[],
   private slices: RangeMapList<Key, DirName> = RangeMapList[],
   private reducer: ?Reducer = None(),
+  /* dirty are keys of files that are made dirty because of a Map */
+  private dirty: SortedSet<Key> = SortedSet[],
   private purgeCount: Int = 0,
   private tombLimit: ?Tick = None(),
   optOnDelete: ?Postponable = None(),
@@ -1535,9 +1537,12 @@ class EagerDir protected {
     }
   }
 
+  fun resetDirty(): this {
+    this with {dirty => SortedSet[]}
+  }
+
   static fun update(
     context: mutable Context,
-    parentDirtyOpt: ?SortedSet<Key>,
     contextDirtyReadersOpt: ?SortedMap<DirName, SortedSet<Key>>,
     parentName: DirName,
     parentMaps: Array<
@@ -1574,10 +1579,8 @@ class EagerDir protected {
         })
       }
     };
-    parentDirtyOpt match {
-    | None() -> void
-    | Some(dirtyKeys) ->
-      keys = dirtyKeys.values();
+    if (!parent.dirty.isEmpty()) {
+      keys = parent.dirty.values();
       !dirty = withRegionFold(None(), keys, dirty, (_, key, dirtyInRegion) ~> {
         for (p in parentMaps) {
           (_, rangeOpt, _) = p;
@@ -1768,7 +1771,8 @@ class EagerDir protected {
       Some(cdata)
     };
 
-    context.addDirty(this.dirName, k);
+    context.addDirWithDirty(this.dirName);
+    dirty = this.dirty.set(k);
     path = Path::create(this.dirName, k);
 
     if (this.totalSize != totalSize) {
@@ -1782,7 +1786,7 @@ class EagerDir protected {
     filesPath = Path::filesTag(this.dirName);
     context.updateDirtyReaders(filesPath);
 
-    !this = this with {totalSize, reducer};
+    !this = this with {dirty, totalSize, reducer};
 
     context.updateDirtyReaders(path);
 

sql/ts/tests/node.play.ts

@@ -17,7 +17,7 @@ tests(true).forEach((t) => {
   run(t, true);
 });
 
-const N = 765000; // we should be able to process this many rows without running out of address space
+const N = 638284; // we should be able to process this many rows without running out of address space
 
 // this is to detect memory regression in wasm, it's not a behaviour
 // test. running only in node as I saw timeout flakiness with firefox.