winget-source: sync metadata using V2 index (#120)

* winget-source: sync metadata required by V2 index

* winget-source: extract common logic of syncing

* update

* add success log

* winget-source: trim trailing spaces

* winget-source: fix unexpected `undefined` in sync

* winget-source: bump package version

winget-source/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ustcmirror/winget-source",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Sync with pre-indexed WinGet source repository.",
   "main": "sync-repo.js",
   "author": "YR Chen <[email protected]>",
@@ -13,7 +13,8 @@
     "node-fetch": "^3.3.1",
     "promised-sqlite3": "^2.1.0",
     "sqlite3": "^5.1.5",
-    "winston": "^3.8.2"
+    "winston": "^3.8.2",
+    "yaml": "^2.4.5"
   },
   "scripts": {
     "start": "node sync-repo.js"

winget-source/sync-repo.js

@@ -3,72 +3,111 @@ import async from 'async'
 
 import { rm } from 'fs/promises'
 import { AsyncDatabase } from 'promised-sqlite3'
-import { EX_IOERR, EX_OK, EX_SOFTWARE, EX_TEMPFAIL, EX_UNAVAILABLE } from './sysexits.js'
+import { EX_IOERR, EX_SOFTWARE, EX_TEMPFAIL, EX_UNAVAILABLE } from './sysexits.js'
 
 import {
+    buildManifestURIs,
+    buildManifestURIsFromPackageMetadata,
+    buildPackageMetadataURIs,
     buildPathpartMap,
-    buildURIList,
+    cacheFileWithURI,
     exitWithCode,
     extractDatabaseFromBundle,
-    getLocalPath,
     makeTempDirectory,
-    saveFile,
     setupEnvironment,
-    syncFile
+    syncFile,
 } from './utilities.js'
 
 
-const sourceV1Filename = 'source.msix';
-const sourceV2Filename = 'source2.msix';
-
-// set up configs and temp directory
 const { parallelLimit, remote, sqlite3, winston } = setupEnvironment();
-const tempDirectory = await makeTempDirectory('winget-repo-');
-
-winston.info(`start syncing with ${remote}`);
-
-try {
-    // download V1 index package to buffer
-    const [indexBuffer, modifiedDate, updated] = await syncFile(sourceV1Filename, true, false);
-    if (!updated) {
-        winston.info(`nothing to sync from ${remote}`);
-        exitWithCode(EX_OK);
-    }
-    assert(indexBuffer !== null, "Failed to get the source index buffer!");
 
-    // unpack, extract and load index database
+/**
+ * Sync with the official WinGet repository index.
+ *
+ * @param {number} version WinGet index version to sync.
+ * @param {(db: AsyncDatabase) => Promise<void>} handler Handler function that reads the index database and syncs necessary files.
+ *
+ * @returns {Promise<void>} Fulfills with `undefined` upon success.
+ */
+async function syncIndex(version, handler) {
+    const tempDirectory = await makeTempDirectory('winget-repo-');
+    const sourceFilename = version > 1 ? `source${version}.msix` : 'source.msix';
     try {
-        const databaseFilePath = await extractDatabaseFromBundle(indexBuffer, tempDirectory);
-        const rawDatabase = new sqlite3.Database(databaseFilePath, sqlite3.OPEN_READONLY);
+        // download index package to buffer
+        const [indexBuffer, modifiedDate, updated] = await syncFile(sourceFilename, true, false);
+        if (!updated) {
+            winston.info(`skip syncing version ${version} from ${remote}`);
+            return;
+        }
+        assert(indexBuffer !== null, "Failed to get the source index buffer!");
 
-        // read manifest URIs from index database
+        // unpack, extract and load index database
         try {
-            const db = new AsyncDatabase(rawDatabase)
-            const pathparts = buildPathpartMap(await db.all('SELECT * FROM pathparts'));
-            const uris = buildURIList(await db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC'), pathparts);
-            await db.close()
-
-            // sync latest manifests in parallel
+            const databaseFilePath = await extractDatabaseFromBundle(indexBuffer, tempDirectory);
+            const database = new sqlite3.Database(databaseFilePath, sqlite3.OPEN_READONLY);
             try {
-                await async.eachLimit(uris, parallelLimit, async (uri) => await syncFile(uri, false));
+                // sync files with handler
+                const asyncDatabase = new AsyncDatabase(database);
+                await handler(asyncDatabase);
+                await asyncDatabase.close();
             } catch (error) {
-                exitWithCode(EX_TEMPFAIL, error);
+                exitWithCode(EX_SOFTWARE, error);
             }
         } catch (error) {
-            exitWithCode(EX_SOFTWARE, error);
+            exitWithCode(EX_IOERR, error);
         }
+
+        // update index package
+        await cacheFileWithURI(sourceFilename, indexBuffer, modifiedDate);
     } catch (error) {
-        exitWithCode(EX_IOERR, error);
+        try {
+            await rm(tempDirectory, { recursive: true });
+        } finally {
+            exitWithCode(EX_UNAVAILABLE, error);
+        }
     }
-
-    // update index packages
-    await saveFile(getLocalPath(sourceV1Filename), indexBuffer, modifiedDate);
-    await syncFile(sourceV2Filename, true);
-} catch (error) {
-    exitWithCode(EX_UNAVAILABLE, error);
+    winston.info(`successfully synced version ${version} from ${remote}`);
+    await rm(tempDirectory, { recursive: true });
 }
 
-winston.info(`successfully synced with ${remote}`);
+winston.info(`start syncing with ${remote}`);
+
+await syncIndex(2, async (db) => {
+    try {
+        const packageURIs = buildPackageMetadataURIs(await db.all('SELECT id, hash FROM packages'));
+        try {
+            // sync latest package metadata in parallel
+            const manifestURIs = await async.concatLimit(packageURIs, parallelLimit, async (uri) => {
+                const [metadataBuffer] = await syncFile(uri, false);
+                try {
+                    return metadataBuffer ? await buildManifestURIsFromPackageMetadata(metadataBuffer) : [];
+                } catch (error) {
+                    exitWithCode(EX_SOFTWARE, error);
+                }
+            });
+            // sync latest manifests in parallel
+            await async.eachLimit(manifestURIs, parallelLimit, async (uri) => await syncFile(uri, false));
+        } catch (error) {
+            exitWithCode(EX_TEMPFAIL, error);
+        }
+    } catch (error) {
+        exitWithCode(EX_SOFTWARE, error);
+    }
+});
 
-// clean up temp directory
-await rm(tempDirectory, { recursive: true });
+await syncIndex(1, async (db) => {
+    try {
+        const pathparts = buildPathpartMap(await db.all('SELECT * FROM pathparts'));
+        const uris = buildManifestURIs(await db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC'), pathparts);
+        // sync latest manifests in parallel
+        try {
+            await async.eachLimit(uris, parallelLimit, async (uri) => await syncFile(uri, false));
+        } catch (error) {
+            exitWithCode(EX_TEMPFAIL, error);
+        }
+    } catch (error) {
+        exitWithCode(EX_SOFTWARE, error);
+    }
+});
+
+winston.info(`successfully synced with ${remote}`);