consume rows one at a time

odota · Dec 22, 2023 · ef42ae2 · ef42ae2
1 parent bdee1ec
commit ef42ae2
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/dev/streamArchive.ts b/dev/streamArchive.ts
@@ -3,9 +3,12 @@ import db from '../store/db';
 import { doArchiveFromLegacy } from '../store/queries';
 
 const stream = db.raw('SELECT match_id from parsed_matches WHERE is_archived IS NULL').stream();
-stream.on('data', async (row) => {
-  await doArchiveFromLegacy(row.match_id.toString());
-  // console.log(row);
+stream.on('readable', async () => {
+  let row;
+  while(row = stream.read()) {
+    // console.log(row);
+    await doArchiveFromLegacy(row.match_id.toString());
+  }
 });
 
 let i = 0;

diff --git a/store/db.ts b/store/db.ts
@@ -9,7 +9,7 @@ const db = knex({
   connection: config.POSTGRES_URL,
   pool: {
     min: 0,
-    max: 40,
+    max: 20,
     // afterCreate: (conn, done) => {
     //   // Set the minimum similarity for pg_trgm
     //   conn.query('SELECT set_limit(0.6);', (err) => {