improvement(build-tools): support 1-deep transitive deps (#23863)

In policy checks transitive dependencies are not deeply considered. But
one level deep checks are now added to handle regular tasks that are
expanded to be complex (multiple tasks).

### Example

Splitting common `build:esnext` into two steps should not require
downstream consumers to change dependencies as dependency on
`build:esnext` still ensures that all of the build steps are run. The
policy checkers know that `build:esnext:main` and/or
`build:esnext:experimental` are required, but not that `build:esnext`
ensures they are run.
```diff
-		"build:esnext": "tsc --project ./tsconfig.json",
+		"build:esnext": "npm run build:esnext:main && npm run build:esnext:experimental",
+		"build:esnext:experimental": "tsc --project ./tsconfig.json",
+		"build:esnext:main": "tsc --project ./tsconfig.main.json",
```

### Infrastructure changes in support:

- Add "children" to Task Definitions to understand what other tasks are
run when running a task. This enables policy checks to work with a
multi-step task as a dependency and not requiring explicit dependency on
one or each part.

"children" property of Task Definition is computed from script command
line explicitly and is not manually configurable. Scripts listed via
`npm run` (no other arguments) and `concurrently` are recognized.

- Refactors existing logic from taskFactory to support `concurrently`
parsing.

- Add broad `readonly` attribute to Task Definitions and Configs that
shouldn't be changed. Add cloning where mutation may occur. (Note that
TypeScript doesn't detect all cases where something readonly is assigned
to non-readonly context. See
microsoft/TypeScript#18770.) Where mutability
is needed in policy fix-up handler, cast away to writeable.

Author: jason-ha

build-tools/packages/build-cli/src/library/repoPolicyCheck/fluidBuildTasks.ts

@@ -300,6 +300,15 @@ function isFluidBuildEnabled(root: string, json: Readonly<PackageJson>): boolean
 	return getFluidPackageMap(root).get(json.name) !== undefined;
 }
 
+function getOrSet<T>(map: Map<string, T>, key: string, defaultValue: T): T {
+	const value = map.get(key);
+	if (value !== undefined) {
+		return value;
+	}
+	map.set(key, defaultValue);
+	return defaultValue;
+}
+
 /**
  * Check if a task has a specific dependency
  * @param root - directory of the Fluid repo root
@@ -316,12 +325,25 @@ function hasTaskDependency(
 ): boolean {
 	const rootConfig = getFluidBuildConfig(root);
 	const globalTaskDefinitions = normalizeGlobalTaskDefinitions(rootConfig?.tasks);
-	const taskDefinitions = getTaskDefinitions(json, globalTaskDefinitions, false);
+	const taskDefinitions = getTaskDefinitions(json, globalTaskDefinitions, {
+		isReleaseGroupRoot: false,
+	});
 	// Searched deps that are package specific (e.g. <packageName>#<taskName>)
 	// It is expected that all packageNames are other packages' names; using
 	// given package's name (json.name) will alway return false as package is
 	// not a dependency of itself. Skip "name# prefix for self dependencies.
 	const packageSpecificSearchDeps = searchDeps.filter((d) => d.includes("#"));
+	const secondaryPackagesTasksToConsider = new Map<
+		string,
+		{ searchDeps: string[]; tasks: Set<string> }
+	>();
+	for (const d of packageSpecificSearchDeps) {
+		const [pkg, task] = d.split("#");
+		getOrSet(secondaryPackagesTasksToConsider, pkg, {
+			searchDeps: [],
+			tasks: new Set<string>(),
+		}).searchDeps.push(task);
+	}
 	/**
 	 * Set of package dependencies
 	 */
@@ -352,31 +374,73 @@ function hasTaskDependency(
 		if (dep.startsWith("^")) {
 			// ^ means "depends on the task of the same name in all package dependencies".
 			// dep of exactly ^* means "_all_ tasks in all package dependencies".
-			const regexSearchMatches = new RegExp(dep === "^*" ? "." : `#${dep.slice(1)}$`);
+			const depPattern = dep.slice(1);
+			const regexSearchMatches = new RegExp(depPattern === "*" ? "." : `#${depPattern}$`);
+			// Check for task matches
 			const possibleSearchMatches = packageSpecificSearchDeps.filter((searchDep) =>
 				regexSearchMatches.test(searchDep),
 			);
+			// Check if there is matching dependency
 			if (
 				possibleSearchMatches.some((searchDep) =>
 					packageDependencies.has(searchDep.split("#")[0]),
 				)
 			) {
 				return true;
 			}
+			if (depPattern === "*") {
+				// No possible match even through transitive dependencies since
+				// ^* would already consider all tasks in all dependencies.
+				continue;
+			}
+			for (const [packageName, secondaryData] of secondaryPackagesTasksToConsider) {
+				// If there is a matching dependency package, add this task to
+				// transitive dependency in secondary package search list.
+				if (packageDependencies.has(packageName)) {
+					secondaryData.tasks.add(depPattern);
+				}
+			}
 			continue;
 		}
-		if (dep.includes("#")) {
-			// Current "pending" dependency is from another package and could possibly
-			// be successor to given queried deps, but we are not doing such a deep or
-			// exhaustive search at this time.
-			continue;
+		const packageDepMatch = dep.match(/^([^#]*)#(.*)$/);
+		if (packageDepMatch) {
+			// Consider one level deep of package's tasks to handle multi-task dependencies.
+			const secondaryPackageSet = secondaryPackagesTasksToConsider.get(packageDepMatch[1]);
+			if (secondaryPackageSet) {
+				secondaryPackageSet.tasks.add(packageDepMatch[2]);
+			}
+		} else {
+			// Do expand transitive dependencies and child tasks from local tasks.
+			const taskDef = taskDefinitions[dep];
+			if (taskDef !== undefined) {
+				pending.push(...taskDef.dependsOn, ...taskDef.children);
+			}
+		}
+	}
+
+	// Consider secondary package dependencies transitive dependencies
+	const packageMap = getFluidPackageMap(root);
+	for (const [packageName, secondaryData] of secondaryPackagesTasksToConsider.entries()) {
+		const pkgJson = packageMap.get(packageName)?.packageJson;
+		if (pkgJson === undefined) {
+			throw new Error(`Dependent package ${packageName} not found in repo`);
 		}
-		// Do expand transitive dependencies from local tasks.
-		// eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
-		if (taskDefinitions[dep]) {
-			pending.push(...taskDefinitions[dep].dependsOn);
+		const secondaryTaskDefinitions = getTaskDefinitions(pkgJson, globalTaskDefinitions, {
+			isReleaseGroupRoot: false,
+		});
+		pending.push(...secondaryData.tasks);
+		let dep;
+		while ((dep = pending.pop()) !== undefined) {
+			if (secondaryData.searchDeps.includes(dep)) {
+				return true;
+			}
+			const taskDef = secondaryTaskDefinitions[dep];
+			if (taskDef !== undefined) {
+				pending.push(...taskDef.dependsOn, ...taskDef.children);
+			}
 		}
 	}
+
 	return false;
 }
 
@@ -408,6 +472,19 @@ function checkTaskDeps(
 		: undefined;
 }
 
+/**
+ * Recursive inverse of Readonly
+ * Makes all properties writeable through entire structure.
+ */
+type DeeplyMutable<T> = { -readonly [K in keyof T]: DeeplyMutable<T[K]> };
+
+/**
+ * Reinterprets a readonly object as a mutable object
+ */
+function asWriteable<T>(onlyReadable: T): DeeplyMutable<T> {
+	return onlyReadable as DeeplyMutable<T>;
+}
+
 /**
  * Fix up the actual dependencies of a task against an expected set of dependent tasks
  * @param root - directory of the Fluid repo root
@@ -428,30 +505,33 @@ function patchTaskDeps(
 	);
 
 	if (missingTaskDependencies.length > 0) {
-		const fileDep = json.fluidBuild?.tasks?.[taskName];
-		if (fileDep === undefined) {
-			let tasks: Exclude<Exclude<PackageJson["fluidBuild"], undefined>["tasks"], undefined>;
+		const readonlyFileDep = json.fluidBuild?.tasks?.[taskName];
+		if (readonlyFileDep === undefined) {
+			let tasks: DeeplyMutable<
+				Exclude<Exclude<PackageJson["fluidBuild"], undefined>["tasks"], undefined>
+			>;
 			if (json.fluidBuild === undefined) {
 				tasks = {};
 				json.fluidBuild = { tasks, version: 1 };
 			} else if (json.fluidBuild.tasks === undefined) {
 				tasks = {};
 				json.fluidBuild.tasks = tasks;
 			} else {
-				tasks = json.fluidBuild.tasks;
+				tasks = asWriteable(json.fluidBuild.tasks);
 			}
 
 			tasks[taskName] = taskDeps.map((dep) => {
 				if (Array.isArray(dep)) {
 					throw new TypeError(
-						`build-tools patchTaskDeps for ${taskName} will not auto select single dependency from choice of ${dep.join(
+						`build-cli patchTaskDeps for ${taskName} will not auto select single dependency from choice of ${dep.join(
 							" or ",
 						)}`,
 					);
 				}
 				return dep;
 			});
 		} else {
+			const fileDep = asWriteable(readonlyFileDep);
 			let depArray: string[];
 			if (Array.isArray(fileDep)) {
 				depArray = fileDep;
@@ -464,12 +544,12 @@ function patchTaskDeps(
 			for (const missingDep of missingTaskDependencies) {
 				if (Array.isArray(missingDep)) {
 					throw new TypeError(
-						`build-tools patchTaskDeps for ${taskName} will not auto select single dependency from choice of ${missingDep.join(
+						`build-cli patchTaskDeps for ${taskName} will not auto select single dependency from choice of ${missingDep.join(
 							" or ",
 						)}`,
 					);
 				}
-				// Check if already added in previous interation to avoid duplicates.
+				// Check if already added in previous iteration to avoid duplicates.
 				if (!depArray.includes(missingDep)) {
 					depArray.push(missingDep);
 				}

build-tools/packages/build-tools/src/fluidBuild/buildGraph.ts

@@ -100,11 +100,9 @@ export class BuildPackage {
 		public readonly pkg: Package,
 		globalTaskDefinitions: TaskDefinitions,
 	) {
-		this._taskDefinitions = getTaskDefinitions(
-			this.pkg.packageJson,
-			globalTaskDefinitions,
-			this.pkg.isReleaseGroupRoot,
-		);
+		this._taskDefinitions = getTaskDefinitions(this.pkg.packageJson, globalTaskDefinitions, {
+			isReleaseGroupRoot: this.pkg.isReleaseGroupRoot,
+		});
 		traceTaskDef(
 			`${pkg.nameColored}: Task def: ${JSON.stringify(this._taskDefinitions, undefined, 2)}`,
 		);
@@ -159,6 +157,7 @@ export class BuildPackage {
 				dependsOn: [`^${taskName}`],
 				script: false,
 				before: [],
+				children: [],
 				after: [],
 			};
 		}
@@ -275,7 +274,7 @@ export class BuildPackage {
 	}
 
 	// Create or get the task with names in the `deps` array
-	private getMatchedTasks(deps: string[], pendingInitDep?: Task[]) {
+	private getMatchedTasks(deps: readonly string[], pendingInitDep?: Task[]) {
 		const matchedTasks: Task[] = [];
 		for (const dep of deps) {
 			// If pendingInitDep is undefined, that mean we don't expect the task to be found, so pretend that we already found it.
@@ -354,7 +353,7 @@ export class BuildPackage {
 		};
 
 		// Expand the star entry to all scheduled tasks
-		const expandStar = (deps: string[], getTaskNames: () => string[]) => {
+		const expandStar = (deps: readonly string[], getTaskNames: () => string[]) => {
 			const newDeps = deps.filter((dep) => dep !== "*");
 			if (newDeps.length === deps.length) {
 				return newDeps;