generate_manifest_urls.mjs

import { readFile, writeFile } from "node:fs/promises";
import { resolve as pathResolve } from "node:path";
import { latestEcmaVersion, parse } from "espree";
import { Syntax, traverse } from "estraverse";
import "./dump_javascript_paths.mjs"; // fixing estraverse Syntax and VisitorKeys

const __dirname = import.meta.dirname;

const files = [
	{
		file: pathResolve(__dirname, "store.steampowered.com/public/javascript/applications/store/manifest.js"),
		cdn: "https://store.steampowered.com/public/",
	},
	{
		file: pathResolve(__dirname, "steamcommunity.com/public/javascript/applications/community/manifest.js"),
		cdn: "https://steamcommunity.com/public/",
	},
	{
		file: pathResolve(__dirname, "help.steampowered.com/public/javascript/applications/help/manifest.js"),
		cdn: "https://help.steampowered.com/public/",
	},
	{
		file: pathResolve(__dirname, "partner.steamgames.com/public/javascript/applications/appmgmt/manifest.js"),
		cdn: "https://partner.steamgames.com/public/",
	},
	/* different AST
	{
		file: pathResolve(__dirname, "partner.steamgames.com/public/javascript/webui/storeadmin/storeadmin.js"),
		cdn: "https://partner.steamgames.com/public/",
	},
	*/
];

const urls = new Set();
let oldUrls = [];

try {
	oldUrls = (await readFile(pathResolve(__dirname, ".support/urls_from_manifests.txt"), { encoding: "utf8" }))
		.trim()
		.split("\n");
} catch {
	// file does not exist
}

for (const url of oldUrls) {
	const fileName = url.substring(url.lastIndexOf("/") + 1);

	if (!/^(chunk~|libraries~|[0-9]+\.)/.test(fileName)) {
		urls.add(url);
	}
}

console.log("Remembered", urls.size, "urls");

for (const { file, cdn } of files) {
	let found = false;

	try {
		const code = await readFile(file);
		const ast = parse(code, { ecmaVersion: latestEcmaVersion, loc: true });

		const seenIds = new Set();

		traverse(ast, {
			enter: function (node) {
				if (
					node.type === Syntax.BinaryExpression &&
					node.left.type === Syntax.BinaryExpression &&
					node.left.right.type === Syntax.Literal &&
					/\.(css|js)\?contenthash=/.test(node.left.right.value) &&
					node.right.type === Syntax.MemberExpression &&
					node.right.object.type === Syntax.ObjectExpression
				) {
					for (const property of node.right.object.properties) {
						if (property.value.type === Syntax.Literal) {
							seenIds.add(property.key.raw);
						}
					}
				}

				// Useful: https://astexplorer.net/
				if (
					node.type === Syntax.BinaryExpression &&
					node.left.type === Syntax.Literal &&
					((node.right.type === Syntax.MemberExpression && node.right.object.type === Syntax.ObjectExpression) ||
						(node.right.type === Syntax.LogicalExpression &&
							node.right.left.type === Syntax.MemberExpression &&
							node.right.left.object.type === Syntax.ObjectExpression))
				) {
					const folder = node.left.value;
					let suffix;

					if (folder.startsWith("javascript/")) {
						suffix = ".js?__TIME__&l=english&__CDN__";
					} else if (folder.startsWith("css/")) {
						suffix = ".css?__TIME__&__CDN__";
					} else {
						console.error("Wrong match:", folder);
						return;
					}

					const partialFileNames = node.right.type === Syntax.LogicalExpression;
					const obj = partialFileNames ? node.right.left.object : node.right.object;

					for (const property of obj.properties) {
						if (property.value.type === Syntax.Literal) {
							const name = property.value.value;

							seenIds.delete(property.key.raw);

							if (name.endsWith("-json") && !name.endsWith("_english-json")) {
								continue;
							}

							const fullUrl = cdn + folder + name + suffix;

							urls.add(fullUrl);

							found = true;
						}
					}

					for (const id of seenIds) {
						const fullUrl = cdn + folder + id + suffix;

						urls.add(fullUrl);
					}

					seenIds.clear();
					this.skip();
				}
			},
		});
	} catch (e) {
		console.error(`Unable to parse "${file}": ${e}`);
		continue;
	}

	if (!found) {
		console.error(`Did not find any files in "${file}"`);
	}
}

console.log("Found", urls.size, "urls");

const strings = [...urls.values()].sort().join("\n") + "\n";

await writeFile(pathResolve(__dirname, ".support/urls_from_manifests.txt"), strings);