Skip to content

Commit 547824d

Browse files
committed
Add catalog-server endpoint to update packages
1 parent 7aa327b commit 547824d

File tree

6 files changed

+157
-4
lines changed

6 files changed

+157
-4
lines changed

packages/catalog-server/src/lib/catalog.ts

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ const toTemporalInstant = (date: Date) => {
3939
*/
4040
const defaultPackageRefreshInterval = Temporal.Duration.from({minutes: 5});
4141

42+
/**
43+
* The default amount of time between automated bulk updates of packages.
44+
*/
45+
const defaultPackageUpdateInterval = Temporal.Duration.from({hours: 6});
46+
4247
export interface CatalogInit {
4348
repository: Repository;
4449
files: PackageFiles;
@@ -74,7 +79,7 @@ export class Catalog {
7479
packageVersion?: PackageVersion;
7580
problems?: ValidationProblem[];
7681
}> {
77-
console.log('Catalog.importPackage');
82+
console.log('Catalog.importPackage', packageName);
7883

7984
const currentPackageInfo = await this.#repository.getPackageInfo(
8085
packageName
@@ -323,4 +328,17 @@ export class Catalog {
323328
// to the repository
324329
return this.#repository.queryElements({query, limit});
325330
}
331+
332+
async getPackagesToUpdate(notUpdatedSince?: Temporal.Instant) {
333+
if (notUpdatedSince === undefined) {
334+
const now = Temporal.Now.instant();
335+
notUpdatedSince = now.subtract(defaultPackageUpdateInterval);
336+
}
337+
338+
const packages = await this.#repository.getPackagesToUpdate(
339+
notUpdatedSince,
340+
100
341+
);
342+
return packages;
343+
}
326344
}

packages/catalog-server/src/lib/firestore/firestore-repository.ts

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
Query,
1010
CollectionReference,
1111
CollectionGroup,
12+
Timestamp,
1213
} from '@google-cloud/firestore';
1314
import {Firestore} from '@google-cloud/firestore';
1415
import firebase from 'firebase-admin';
@@ -42,6 +43,7 @@ import {
4243
import {packageVersionConverter} from './package-version-converter.js';
4344
import {customElementConverter} from './custom-element-converter.js';
4445
import {validationProblemConverter} from './validation-problem-converter.js';
46+
import type {Temporal} from '@js-temporal/polyfill';
4547

4648
const projectId = 'wc-catalog';
4749
firebase.initializeApp({projectId});
@@ -523,13 +525,38 @@ export class FirestoreRepository implements Repository {
523525
return result;
524526
}
525527

526-
getPackageRef(packageName: string) {
528+
async getPackagesToUpdate(
529+
notUpdatedSince: Temporal.Instant,
530+
limit = 100
531+
): Promise<Array<PackageInfo>> {
532+
533+
const date = new Date(notUpdatedSince.epochMilliseconds);
534+
const notUpdatedSinceTimestamp = Timestamp.fromDate(date);
535+
536+
// Only query 'READY', 'ERROR', and 'NOT_FOUND' packages.
537+
// INITIALIZING and UPDATING packages are being updated, possibly by the
538+
// batch update task calling this method.
539+
// ERROR and NOT_FOUND are "recoverable" errors, so we should try to import
540+
// them again.
541+
const result = await this.getPackageCollectionRef()
542+
.where('status', 'in', ['READY', 'ERROR', 'NOT_FOUND'])
543+
.where('lastUpdate', '<', notUpdatedSinceTimestamp)
544+
.limit(limit)
545+
.get();
546+
const packages = result.docs.map((d) => d.data());
547+
return packages;
548+
}
549+
550+
getPackageCollectionRef() {
527551
return db
528552
.collection('packages' + (this.namespace ? `-${this.namespace}` : ''))
529-
.doc(packageNameToId(packageName))
530553
.withConverter(packageInfoConverter);
531554
}
532555

556+
getPackageRef(packageName: string) {
557+
return this.getPackageCollectionRef().doc(packageNameToId(packageName));
558+
}
559+
533560
getPackageVersionCollectionRef(packageName: string) {
534561
return this.getPackageRef(packageName)
535562
.collection('versions')

packages/catalog-server/src/lib/repository.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
* SPDX-License-Identifier: Apache-2.0
55
*/
66

7+
import type {Temporal} from '@js-temporal/polyfill';
78
import type {
89
CustomElement,
910
PackageInfo,
@@ -146,4 +147,12 @@ export interface Repository {
146147
packageName: string,
147148
version: string
148149
): Promise<PackageVersion | undefined>;
150+
151+
/**
152+
* Returns packages that have not been updated since the date given.
153+
*/
154+
getPackagesToUpdate(
155+
notUpdatedSince: Temporal.Instant,
156+
limit: number
157+
): Promise<Array<PackageInfo>>;
149158
}

packages/catalog-server/src/lib/server/routes/bootstrap-packages.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ export const makeBootstrapPackagesRoute =
2222
const bootstrapListFile = await readFile(bootstrapListFilePath, 'utf-8');
2323
const bootstrapList = JSON.parse(bootstrapListFile);
2424
const packageNames = bootstrapList['packages'] as Array<string>;
25+
26+
// TODO (justinfagnani): rather than import the packages directly, add them
27+
// to the DB in a non-imported state, then kick off the standard update
28+
// workflow, which will import them all.
2529
const results = await Promise.all(
2630
packageNames.map(
2731
async (
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import {Temporal} from '@js-temporal/polyfill';
2+
import {PackageInfo} from '@webcomponents/catalog-api/lib/schema.js';
3+
import type Koa from 'koa';
4+
import type {Catalog} from '../../catalog.js';
5+
6+
// Google Cloud Run default request timeout is 5 minutes, so to do longer
7+
// imports we need to configure the timeout.
8+
const maxImportDuration = Temporal.Duration.from({minutes: 5});
9+
10+
export const makeUpdatePackagesRoute =
11+
(catalog: Catalog) => async (context: Koa.Context) => {
12+
const startInstant = Temporal.Now.instant();
13+
// If the `force` query parameter is present we force updating of all
14+
// packages by setting the `notUpdatedSince` parameter to `startInstant` so
15+
// that we get all packages last updated before now. We calculate the
16+
// `notUpdatedSince` time once before updates so that we don't retrieve
17+
// packages that we update in this operation.
18+
// `force`` is useful for development and testing as we may be trying to
19+
// update packages that were just imported.
20+
// TODO (justinfagnani): check a DEV mode also so this isn't available
21+
// in production?
22+
const force = 'force' in context.query;
23+
const notUpdatedSince = force ? startInstant : undefined;
24+
25+
// If `force` is true, override the default packageUpdateInterval
26+
// TODO: how do we make an actually 0 duration?
27+
const packageUpdateInterval = force
28+
? Temporal.Duration.from({microseconds: 1})
29+
: undefined;
30+
31+
console.log('Starting package update at', startInstant, `force: ${force}`);
32+
33+
let packagesToUpdate!: Array<PackageInfo>;
34+
let packagesUpdated = 0;
35+
let iteration = 0;
36+
37+
// Loop through batches of packages to update.
38+
// We batch here so that we can pause and check that we're still within the
39+
// maxImportDuration, and use small enough batches so that we can ensure at
40+
// least one batch in that time.
41+
do {
42+
// getPackagesToUpdate() queries the first N (default 100) packages that
43+
// have not been updated since the update interval (default 6 hours).
44+
// When a package is imported it's lastUpdate date will be updated and the
45+
// next call to getPackagesToUpdate() will return the next 100 packages.
46+
// This way we don't need a DB cursor to make progress through the
47+
// package list.
48+
packagesToUpdate = await catalog.getPackagesToUpdate(notUpdatedSince);
49+
50+
if (packagesToUpdate.length === 0) {
51+
// No more packages to update
52+
if (iteration === 0) {
53+
console.log('No packages to update');
54+
}
55+
break;
56+
}
57+
58+
await Promise.allSettled(
59+
packagesToUpdate.map(async (pkg) => {
60+
try {
61+
return await catalog.importPackage(pkg.name, packageUpdateInterval);
62+
} catch (e) {
63+
console.error(e);
64+
throw e;
65+
}
66+
})
67+
);
68+
packagesUpdated += packagesToUpdate.length;
69+
70+
const now = Temporal.Now.instant();
71+
const timeSinceStart = now.since(startInstant);
72+
// If the time since the update started is not less than that max import
73+
// duration, stop.
74+
// TODO (justinfagnani): we need a way to test this
75+
if (Temporal.Duration.compare(timeSinceStart, maxImportDuration) !== -1) {
76+
break;
77+
}
78+
} while (true);
79+
console.log(`Updated ${packagesUpdated} packages`);
80+
81+
if (packagesToUpdate.length > 0) {
82+
// TODO (justinfagnani): kick off new update request
83+
console.log(`Not all packages were updated (${packagesToUpdate.length})`);
84+
}
85+
86+
context.status = 200;
87+
context.type = 'html';
88+
context.body = `
89+
<h1>Update Results</h1>
90+
<p>Updated ${packagesUpdated} package</p>
91+
`;
92+
};

packages/catalog-server/src/lib/server/server.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/**
22
* @license
3-
* Copyright 2021 Google LLC
3+
* Copyright 2022 Google LLC
44
* SPDX-License-Identifier: BSD-3-Clause
55
*/
66

@@ -17,6 +17,7 @@ import {NpmAndUnpkgFiles} from '@webcomponents/custom-elements-manifest-tools/li
1717

1818
import {makeGraphQLRoute} from './routes/graphql.js';
1919
import {makeBootstrapPackagesRoute} from './routes/bootstrap-packages.js';
20+
import {makeUpdatePackagesRoute} from './routes/update-packages.js';
2021

2122
export const makeServer = async () => {
2223
const files = new NpmAndUnpkgFiles();
@@ -32,6 +33,8 @@ export const makeServer = async () => {
3233

3334
router.get('/bootstrap-packages', makeBootstrapPackagesRoute(catalog));
3435

36+
router.get('/update-packages', makeUpdatePackagesRoute(catalog));
37+
3538
router.get('/', async (ctx) => {
3639
ctx.status = 200;
3740
ctx.type = 'html';

0 commit comments

Comments
 (0)