forked from smikula/good-fences
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a file provider that avoids ts program walk (smikula#100)
* Add a file provider that avoids ts program walk Adds a faster file provider that discovers source and fence files with a fast asyncronous file walk rather than by relying on typescript to perform a syncronous walk + parse of the program. We still use typescript to parse compilerOptions, but since we no longer rely on the typescript program walk to identify files, we stub out file discovery so only the initial config partse happens. Also adds config options to support switching between the two providers, since it will only work when all source files you intend to check fences against fall under your rootDirs. This is the case for OWA, but I don't know what other consumers look like. Depends on u/mahuangh/faster-source-providers-2 * progress -> progressBar * progress -> progressBar * support definition files, simplify resolveImportsFromFile, exclude asset files in fdir source file provider * getScriptFileExtensions * add picomatch as a dependency for fdir glob * Move extension set calculation to getScriptFileExtensions Co-authored-by: Maxwell Huang-Hobbs <[email protected]>
- Loading branch information
1 parent
6e9cc52
commit 668999c
Showing
9 changed files
with
308 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,224 @@ | ||
| import { SourceFileProvider } from './SourceFileProvider'; | ||
| import { fdir } from 'fdir'; | ||
| import NormalizedPath from '../types/NormalizedPath'; | ||
| import { promisify } from 'util'; | ||
| import * as fs from 'fs'; | ||
| import * as path from 'path'; | ||
| const readFile = promisify(fs.readFile); | ||
| const stat = promisify(fs.stat); | ||
| import { createMatchPathAsync, MatchPathAsync } from 'tsconfig-paths'; | ||
| import { getScriptFileExtensions } from '../utils/getScriptFileExtensions'; | ||
| import { | ||
| getParsedCommandLineOfConfigFile, | ||
| JsxEmit, | ||
| ParsedCommandLine, | ||
| preProcessFile, | ||
| } from 'typescript'; | ||
|
|
||
| export class FDirSourceFileProvider implements SourceFileProvider { | ||
| parsedCommandLine: ParsedCommandLine; | ||
| matchPath: MatchPathAsync; | ||
| private sourceFileGlob: string; | ||
| private extensionsToCheckDuringImportResolution: string[]; | ||
|
|
||
| constructor(configFileName: NormalizedPath, private rootDirs: string[]) { | ||
| // Load the full config file, relying on typescript to recursively walk the "extends" fields, | ||
| // while stubbing readDirectory calls to stop the full file walk of the include() patterns. | ||
| // | ||
| // We do this because we need to access the parsed compilerOptions, but do not care about | ||
| // the full file list. | ||
| this.parsedCommandLine = getParsedCommandLineOfConfigFile( | ||
| configFileName, | ||
| {}, // optionsToExtend | ||
| { | ||
| getCurrentDirectory: process.cwd, | ||
| fileExists: fs.existsSync, | ||
| useCaseSensitiveFileNames: true, | ||
| readFile: path => fs.readFileSync(path, 'utf-8'), | ||
| readDirectory: () => { | ||
| // this is supposed to be the recursive file walk. | ||
| // since we don't care about _actually_ discovering files, | ||
| // only about parsing the config's compilerOptions | ||
| // (and tracking the "extends": fields across multiple files) | ||
| // we short circuit this. | ||
| return []; | ||
| }, | ||
| onUnRecoverableConfigFileDiagnostic: diagnostic => { | ||
| console.error(diagnostic); | ||
| process.exit(1); | ||
| }, | ||
| } | ||
| ); | ||
|
|
||
| this.sourceFileGlob = `**/*@(${getScriptFileExtensions({ | ||
| // Derive these settings from the typescript project itself | ||
| allowJs: this.parsedCommandLine.options.allowJs || false, | ||
| jsx: this.parsedCommandLine.options.jsx !== JsxEmit.None, | ||
| // Since we're trying to find script files that can have imports, | ||
| // we explicitly exclude json modules | ||
| includeJson: false, | ||
| // since definition files are '.d.ts', the extra | ||
| // definition extensions here are covered by the glob '*.ts' from | ||
| // the above settings. | ||
| // | ||
| // Here as an optimization we avoid adding these definition files while | ||
| // globbing | ||
| includeDefinitions: false, | ||
| }).join('|')})`; | ||
|
|
||
| // Script extensions to check when looking for imports. | ||
| this.extensionsToCheckDuringImportResolution = getScriptFileExtensions({ | ||
| // Derive these settings from the typescript project itself | ||
| allowJs: this.parsedCommandLine.options.allowJs || false, | ||
| jsx: this.parsedCommandLine.options.jsx !== JsxEmit.None, | ||
| includeJson: this.parsedCommandLine.options.resolveJsonModule, | ||
| // When scanning for imports, we always consider importing | ||
| // definition files. | ||
| includeDefinitions: true, | ||
| }); | ||
|
|
||
| this.matchPath = createMatchPathAsync( | ||
| this.parsedCommandLine.options.baseUrl, | ||
| this.parsedCommandLine.options.paths | ||
| ); | ||
| } | ||
|
|
||
| async getSourceFiles(searchRoots?: string[]): Promise<string[]> { | ||
| const allRootsDiscoveredFiles: string[][] = await Promise.all( | ||
| (searchRoots || this.rootDirs).map( | ||
| (rootDir: string) => | ||
| new fdir() | ||
| .glob(this.sourceFileGlob) | ||
| .withFullPaths() | ||
| .crawl(rootDir) | ||
| .withPromise() as Promise<string[]> | ||
| ) | ||
| ); | ||
|
|
||
| return [...new Set<string>(allRootsDiscoveredFiles.reduce((a, b) => a.concat(b), []))]; | ||
| } | ||
|
|
||
| async getImportsForFile(filePath: string): Promise<string[]> { | ||
| const fileInfo = preProcessFile(await readFile(filePath, 'utf-8'), true, true); | ||
| return fileInfo.importedFiles.map(importedFile => importedFile.fileName); | ||
| } | ||
|
|
||
| async resolveImportFromFile( | ||
| importer: string, | ||
| importSpecifier: string | ||
| ): Promise<string | undefined> { | ||
| if (importSpecifier.startsWith('.')) { | ||
| // resolve relative and check extensions | ||
| const directImportResult = await checkExtensions( | ||
| path.join(path.dirname(importer), importSpecifier), | ||
| [ | ||
| ...this.extensionsToCheckDuringImportResolution, | ||
| // Also check for no-exension to permit import specifiers that | ||
| // already have an extension (e.g. require('foo.js')) | ||
| '', | ||
| // also check for directory index imports | ||
| ...this.extensionsToCheckDuringImportResolution.map(x => '/index' + x), | ||
| ] | ||
| ); | ||
|
|
||
| if ( | ||
| directImportResult && | ||
| this.extensionsToCheckDuringImportResolution.some(extension => | ||
| directImportResult.endsWith(extension) | ||
| ) | ||
| ) { | ||
| // this is an allowed script file | ||
| return directImportResult; | ||
| } else { | ||
| // this is an asset file | ||
| return undefined; | ||
| } | ||
| } else { | ||
| // resolve with tsconfig-paths (use the paths map, then fall back to node-modules) | ||
| return await new Promise((resolve, reject) => | ||
| this.matchPath( | ||
| importSpecifier, | ||
| undefined, // readJson | ||
| undefined, // fileExists | ||
| [...this.extensionsToCheckDuringImportResolution, ''], | ||
| async (err: Error, result: string) => { | ||
| if (err) { | ||
| reject(err); | ||
| } else if (!result) { | ||
| resolve(undefined); | ||
| } else { | ||
| if ( | ||
| isFile(result) && | ||
| this.extensionsToCheckDuringImportResolution.some(extension => | ||
| result.endsWith(extension) | ||
| ) | ||
| ) { | ||
| // this is an exact require of a known script extension, resolve | ||
| // it up front | ||
| resolve(result); | ||
| } else { | ||
| // tsconfig-paths returns a path without an extension. | ||
| // if it resolved to an index file, it returns the path to | ||
| // the directory of the index file. | ||
| if (await isDirectory(result)) { | ||
| resolve( | ||
| checkExtensions( | ||
| path.join(result, 'index'), | ||
| this.extensionsToCheckDuringImportResolution | ||
| ) | ||
| ); | ||
| } else { | ||
| resolve( | ||
| checkExtensions( | ||
| result, | ||
| this.extensionsToCheckDuringImportResolution | ||
| ) | ||
| ); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| ) | ||
| ); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| async function isFile(filePath: string): Promise<boolean> { | ||
| try { | ||
| // stat will throw if the file does not exist | ||
| const statRes = await stat(filePath); | ||
| if (statRes.isFile()) { | ||
| return true; | ||
| } | ||
| } catch { | ||
| // file does not exist | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| async function isDirectory(filePath: string): Promise<boolean> { | ||
| try { | ||
| // stat will throw if the file does not exist | ||
| const statRes = await stat(filePath); | ||
| if (statRes.isDirectory()) { | ||
| return true; | ||
| } | ||
| } catch { | ||
| // file does not exist | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| async function checkExtensions( | ||
| filePathNoExt: string, | ||
| extensions: string[] | ||
| ): Promise<string | undefined> { | ||
| for (let ext of extensions) { | ||
| const joinedPath = filePathNoExt + ext; | ||
| if (await isFile(joinedPath)) { | ||
| return joinedPath; | ||
| } | ||
| } | ||
| return undefined; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| export type PartialConfigOptions = { | ||
| allowJs: boolean; | ||
| jsx: boolean; | ||
| includeJson?: boolean; | ||
| includeDefinitions?: boolean; | ||
| }; | ||
|
|
||
| export function getScriptFileExtensions(options: PartialConfigOptions): string[] { | ||
| const extensions: string[] = ['.ts']; | ||
| if (options.allowJs) { | ||
| extensions.push('.js'); | ||
| if (options.jsx) { | ||
| extensions.push('.jsx'); | ||
| } | ||
| } | ||
|
|
||
| if (options.includeJson) { | ||
| extensions.push('.json'); | ||
| } | ||
|
|
||
| if (options.jsx) { | ||
| extensions.push('.tsx'); | ||
| } | ||
|
|
||
| if (options.includeDefinitions) { | ||
| extensions.push('.d.ts'); | ||
| if (options.jsx) { | ||
| // I don't know why this would ever | ||
| // be a thing, but it is, so I'm adding it here. | ||
| extensions.push('.d.jsx'); | ||
| } | ||
| } | ||
|
|
||
| return extensions; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters