Skip to content

Commit 4b95271

Browse files
amoelleramoeller
committed
Moved AST modifications to preprocessor
1 parent 2963f90 commit 4b95271

13 files changed

+167
-92
lines changed

‎src/analysis/analysisstate.ts

+4-5
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ import {
99
isNode,
1010
NewExpression,
1111
Node,
12-
OptionalCallExpression,
13-
SourceLocation
12+
OptionalCallExpression
1413
} from "@babel/types";
1514
import {
1615
FilePath,
@@ -42,7 +41,7 @@ import {ConstraintVarProducer} from "./constraintvarproducer";
4241
import Timer from "../misc/timer";
4342
import {VulnerabilityDetector} from "../patternmatching/vulnerabilitydetector";
4443

45-
export const globalLoc: SourceLocation = {start: {line: 0, column: 0}, end: {line: 0, column: 0}};
44+
export const globalLoc: SourceLocationWithFilename = {start: {line: 1, column: 0}, end: {line: 1, column: 0}, filename: "%global"};
4645

4746
export const undefinedIdentifier = identifier("undefined"); // TODO: prevent writes to 'undefined'?
4847
undefinedIdentifier.loc = globalLoc;
@@ -537,9 +536,9 @@ export class AnalysisState { // TODO: move some of these fields to FragmentState
537536
/**
538537
* Registers a new FunctionInfo for a function/method/constructor.
539538
*/
540-
registerFunctionInfo(file: FilePath, path: NodePath<Function | Class>, name: string | undefined, fun: Function, loc: SourceLocation | null | undefined) {
539+
registerFunctionInfo(file: FilePath, path: NodePath<Function | Class>, name: string | undefined, fun: Function) {
541540
const m = this.moduleInfos.get(file)!;
542-
const f = new FunctionInfo(name, loc, m);
541+
const f = new FunctionInfo(name, fun, m);
543542
this.functionInfos.set(fun, f);
544543
const parent = path.getFunctionParent()?.node;
545544
(parent ? this.functionInfos.get(parent)!.functions : m.functions).set(fun, f);

‎src/analysis/analyzer.ts

+6-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {Operations} from "./operations";
1919
import {Program} from "@babel/types";
2020
import {UnknownAccessPath} from "./accesspaths";
2121
import {Token} from "./tokens";
22+
import {preprocessAst} from "../parsing/extras";
2223

2324
export async function analyzeFiles(files: Array<string>, solver: Solver, returnFileMap: boolean = false): Promise<Map<FilePath, {sourceCode: string, program: Program}>> {
2425
const a = solver.analysisState;
@@ -67,11 +68,14 @@ export async function analyzeFiles(files: Array<string>, solver: Solver, returnF
6768
solver.prepare();
6869

6970
// prepare model of native library
70-
const [globals, specials] = buildNatives(solver, moduleInfo);
71+
const {globals, globalsHidden, specials} = buildNatives(solver, moduleInfo);
72+
73+
// preprocess the AST
74+
preprocessAst(ast, file, globals, globalsHidden);
7175

7276
// traverse the AST
7377
writeStdOutIfActive("Traversing AST...");
74-
visit(ast, new Operations(file, solver, globals, specials));
78+
visit(ast, new Operations(file, solver, specials));
7579

7680
// propagate tokens until fixpoint reached for the module
7781
await solver.propagate();

‎src/analysis/astvisitor.ts

+6-40
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ import {
6363
ObjectProperty,
6464
OptionalCallExpression,
6565
OptionalMemberExpression,
66-
Program,
6766
RegExpLiteral,
6867
ReturnStatement,
6968
SequenceExpression,
@@ -72,8 +71,6 @@ import {
7271
TaggedTemplateExpression,
7372
ThisExpression,
7473
ThrowStatement,
75-
variableDeclaration,
76-
variableDeclarator,
7774
VariableDeclarator,
7875
WithStatement,
7976
YieldExpression
@@ -90,9 +87,8 @@ import {
9087
} from "./tokens";
9188
import {ModuleInfo} from "./infos";
9289
import logger from "../misc/logger";
93-
import {mapArrayAdd, sourceLocationToStringWithFile, SourceLocationWithFilename} from "../misc/util";
90+
import {mapArrayAdd, sourceLocationToStringWithFile} from "../misc/util";
9491
import assert from "assert";
95-
import {globalLoc, undefinedIdentifier} from "./analysisstate";
9692
import {options} from "../options";
9793
import {ComponentAccessPath, PropertyAccessPath, UnknownAccessPath} from "./accesspaths";
9894
import {ConstraintVar} from "./constraintvars";
@@ -118,44 +114,17 @@ export const IDENTIFIER_KIND = Symbol();
118114
export function visit(ast: File, op: Operations) {
119115
const solver = op.solver;
120116
const a = solver.analysisState;
121-
let nextNodeIndex = 1;
122117

123118
a.maybeEscaping.clear(); // TODO: move to method in AnalysisState?
124119

120+
// set module source location
121+
op.moduleInfo.node = ast.program;
122+
125123
// traverse the AST and extend the analysis result with information about the current module
126124
if (logger.isVerboseEnabled())
127125
logger.verbose(`Traversing AST of ${op.file}`);
128126
traverse(ast, {
129127

130-
enter(path: NodePath) {
131-
132-
// workaround to ensure that AST nodes with undefined location (caused by desugaring) can be identified uniquely
133-
if (!path.node.loc) {
134-
let p: NodePath | null = path;
135-
while (p && !p.node.loc)
136-
p = p.parentPath;
137-
path.node.loc = {filename: op.file, start: p?.node.loc?.start, end: p?.node.loc?.end, nodeIndex: nextNodeIndex++} as any; // see sourceLocationToString
138-
}
139-
},
140-
141-
Program(path: NodePath<Program>) {
142-
143-
// set module source location
144-
op.moduleInfo.loc = path.node.loc;
145-
146-
// artificially declare all globals in the program scope (if not already declared)
147-
const decls = [...op.globals, undefinedIdentifier] // TODO: treat 'undefined' like other globals?
148-
.filter(d => path.scope.getBinding(d.name) === undefined)
149-
.map(id => {
150-
const d = variableDeclarator(id);
151-
d.loc = globalLoc;
152-
return d;
153-
});
154-
const d = variableDeclaration("var", decls);
155-
d.loc = globalLoc;
156-
path.scope.registerDeclaration(path.unshiftContainer("body", d)[0]);
157-
},
158-
159128
ThisExpression(path: NodePath<ThisExpression>) {
160129

161130
// this
@@ -249,13 +218,10 @@ export function visit(ast: File, op: Operations) {
249218
(isObjectMethod(path.node) || isClassMethod(path.node)) ? getKey(path.node) :
250219
cls ? cls.id?.name : undefined;// for constructors, use the class name if present
251220
const anon = isFunctionDeclaration(path.node) || isFunctionExpression(path.node) ? path.node.id === null : isArrowFunctionExpression(path.node);
252-
const loc = cls ? cls.loc : // for constructors, use the class location (workaround to match dyn.ts)
253-
isClassMethod(fun) && fun.static ? {filename: (fun.key.loc as SourceLocationWithFilename).filename, start: fun.key.loc!.start, end: fun.loc!.end} : // for static methods, use the identifier location (workaround to match dyn.ts)
254-
fun.loc;
255221
const msg = cls ? "constructor" : `${name ?? (anon ? "<anonymous>" : "<computed>")}`;
256222
if (logger.isVerboseEnabled())
257-
logger.verbose(`Reached function ${msg} at ${sourceLocationToStringWithFile(loc)}`);
258-
a.registerFunctionInfo(op.file, path, name, fun, loc);
223+
logger.verbose(`Reached function ${msg} at ${sourceLocationToStringWithFile(fun.loc)}`);
224+
a.registerFunctionInfo(op.file, path, name, fun);
259225
if (!name && !anon)
260226
a.warnUnsupported(fun, `Computed ${isFunctionDeclaration(path.node) || isFunctionExpression(path.node) ? "function" : "method"} name`); // TODO: handle functions/methods with unknown name?
261227

‎src/analysis/infos.ts

+10-7
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import {FilePath, sourceLocationToString} from "../misc/util";
2-
import {Function, SourceLocation} from "@babel/types";
1+
import {FilePath, sourceLocationToString, strHash} from "../misc/util";
2+
import {Function, Program} from "@babel/types";
33
import {FragmentState} from "./fragmentstate";
44

55
/**
@@ -57,15 +57,18 @@ export class ModuleInfo {
5757

5858
readonly isEntry: boolean; // true for entry modules
5959

60-
loc: SourceLocation | null | undefined = null; // top-level source location (set by astvisitor)
60+
node: Program | undefined; // top-level source location (set by astvisitor)
6161

6262
fragmentState: FragmentState | undefined = undefined; // analysis solution after local analysis of this module
6363

64+
hash: number;
65+
6466
constructor(relativePath: string, packageInfo: PackageInfo, path: FilePath, isEntry: boolean) {
6567
this.relativePath = relativePath;
6668
this.packageInfo = packageInfo;
6769
this.path = path;
6870
this.isEntry = isEntry;
71+
this.hash = strHash(this.toString());
6972
}
7073

7174
toString(): string {
@@ -110,7 +113,7 @@ export class FunctionInfo {
110113

111114
readonly name: string | undefined; // function name
112115

113-
readonly loc: SourceLocation | null | undefined; // function source location
116+
readonly node: Function; // function source location
114117

115118
readonly moduleInfo: ModuleInfo; // module containing this function
116119

@@ -120,13 +123,13 @@ export class FunctionInfo {
120123
return this.moduleInfo.packageInfo;
121124
}
122125

123-
constructor(name: string | undefined, loc: SourceLocation | null | undefined, moduleInfo: ModuleInfo) {
126+
constructor(name: string | undefined, node: Function, moduleInfo: ModuleInfo) {
124127
this.name = name;
125-
this.loc = loc;
128+
this.node = node;
126129
this.moduleInfo = moduleInfo;
127130
}
128131

129132
toString() {
130-
return `${this.moduleInfo}:${sourceLocationToString(this.loc)}:${this.name ?? "<anonymous>"}`;
133+
return `${this.moduleInfo}:${sourceLocationToString(this.node.loc)}:${this.name ?? "<anonymous>"}`;
131134
}
132135
}

‎src/analysis/operations.ts

+1-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import {
22
CallExpression,
33
Expression,
44
Function,
5-
Identifier,
65
isArrayPattern,
76
isAssignmentPattern,
87
isExportDeclaration,
@@ -80,8 +79,6 @@ export class Operations {
8079

8180
readonly solver: Solver;
8281

83-
readonly globals: Array<Identifier>;
84-
8582
readonly natives: SpecialNativeObjects
8683

8784
readonly a: AnalysisState; // shortcut to this.solver.analysisState
@@ -96,10 +93,9 @@ export class Operations {
9693

9794
readonly exportsObjectToken: NativeObjectToken;
9895

99-
constructor(file: FilePath, solver: Solver, globals: Array<Identifier>, natives: SpecialNativeObjects) {
96+
constructor(file: FilePath, solver: Solver, natives: SpecialNativeObjects) {
10097
this.file = file;
10198
this.solver = solver;
102-
this.globals = globals;
10399
this.natives = natives;
104100

105101
this.a = this.solver.analysisState;

‎src/analysis/solver.ts

+4-9
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import Timer from "../misc/timer";
3737
import {setImmediate} from "timers/promises";
3838
import {AnalysisDiagnostics} from "../typings/diagnostics";
3939
import {getMemoryUsage} from "../misc/memory";
40+
import {JELLY_NODE_ID} from "../parsing/extras";
4041

4142
export class AbortedException extends Error {}
4243

@@ -50,9 +51,6 @@ export default class Solver {
5051

5152
unprocessedSubsetEdges: Map<ConstraintVar, Set<ConstraintVar>> = new Map;
5253

53-
idSymbol = Symbol();
54-
nextNodeID = 0;
55-
5654
// TODO: move some of this into AnalysisDiagnostics?
5755
// for diagnostics only
5856
unprocessedTokensSize: number = 0;
@@ -335,12 +333,9 @@ export default class Solver {
335333
* Provides a unique ID for the given key and node.
336334
*/
337335
private getListenerID(key: TokenListener, n: Node): ListenerID {
338-
let id = (n as any)[this.idSymbol] as number | undefined;
339-
if (id === undefined) {
340-
id = this.nextNodeID++;
341-
(n as any)[this.idSymbol] = id;
342-
}
343-
return (id << 10) + key;
336+
let id = (n as any)[JELLY_NODE_ID];
337+
assert(id !== undefined);
338+
return ((id << 10) + key) ^ (this.analysisState.moduleInfos.get((n as any)?.loc?.filename)?.hash || 0);
344339
}
345340

346341
/**

‎src/dynamic/soundnesstester.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ export function testSoundness(jsonfile: string, analysisState: AnalysisState): [
1919
// collect all static functions including module top-level functions (excluding aliases)
2020
const staticFunctions = new Map<string, FunctionInfo | ModuleInfo>();
2121
for (const m of analysisState.moduleInfos.values())
22-
if (m.loc) // TODO: m.loc may be empty with --ignore-dependencies
23-
staticFunctions.set(`${m.path}:${m.loc.start.line}:${m.loc.start.column + 1}:${m.loc.end.line}:${m.loc.end.column + 1}`, m);
22+
if (m.node?.loc) // TODO: m.node.loc may be empty with --ignore-dependencies
23+
staticFunctions.set(`${m.path}:${m.node.loc.start.line}:${m.node.loc.start.column + 1}:${m.node.loc.end.line}:${m.node.loc.end.column + 1}`, m);
2424
for (const f of analysisState.functionInfos.values())
25-
if (!f.loc || "nodeIndex" in f.loc)
25+
if (!f.node?.loc || "nodeIndex" in f.node.loc)
2626
analysisState.warn(`Source location missing for function ${f.name || "<anonymous>"} in ${f.moduleInfo.path}`);
2727
else
28-
staticFunctions.set(`${f.moduleInfo.path}:${f.loc.start.line}:${f.loc.start.column + 1}:${f.loc.end.line}:${f.loc.end.column + 1}`, f);
28+
staticFunctions.set(`${f.moduleInfo.path}:${f.node.loc.start.line}:${f.node.loc.start.column + 1}:${f.node.loc.end.line}:${f.node.loc.end.column + 1}`, f);
2929

3030
// log static locations
3131
if (logger.isDebugEnabled()) {

‎src/natives/nativebuilder.ts

+13-13
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,10 @@ export type SpecialNativeObjects = Map<string, NativeObjectToken | PackageObject
6161
* Prepares models for the ECMAScript and Node.js native declarations.
6262
* Returns the global identifiers and the tokens for special native objects.
6363
*/
64-
export function buildNatives(solver: Solver, moduleInfo: ModuleInfo): [Array<Identifier>, SpecialNativeObjects] {
64+
export function buildNatives(solver: Solver, moduleInfo: ModuleInfo): {globals: Array<Identifier>, globalsHidden: Array<Identifier>, specials: SpecialNativeObjects} {
6565
const globals: Array<Identifier> = [];
66-
const natives: SpecialNativeObjects = new Map;
66+
const globalsHidden: Array<Identifier> = [];
67+
const specials: SpecialNativeObjects = new Map;
6768
const a = solver.analysisState;
6869

6970
const models = [ecmascriptModels, nodejsModels];
@@ -84,13 +85,12 @@ export function buildNatives(solver: Solver, moduleInfo: ModuleInfo): [Array<Ide
8485
if (options.natives || m.name === "ecmascript" || (m.name === "nodejs" && ["exports", "module"].includes(name))) {
8586
const id = identifier(name);
8687
id.loc = loc;
87-
if (!hidden)
88-
globals.push(id);
88+
(hidden ? globalsHidden : globals).push(id);
8989
const t = init
90-
? init({solver, moduleInfo, natives, id})
90+
? init({solver, moduleInfo, natives: specials, id})
9191
: a.canonicalizeToken(new NativeObjectToken(name, moduleSpecific ? moduleInfo : undefined, invoke, constr));
9292
solver.addTokenConstraint(t, a.varProducer.nodeVar(id));
93-
natives.set(name, t);
93+
specials.set(name, t);
9494
}
9595
}
9696

@@ -107,9 +107,9 @@ export function buildNatives(solver: Solver, moduleInfo: ModuleInfo): [Array<Ide
107107
function definePrototypeObject(name: string): NativeObjectToken {
108108
const t = a.canonicalizeToken(new NativeObjectToken(`${name}.prototype`));
109109
if (m.name === "ecmascript")
110-
natives.set(t.name, t);
110+
specials.set(t.name, t);
111111
if (options.natives || m.name === "ecmascript")
112-
solver.addTokenConstraint(t, a.varProducer.objPropVar(natives.get(name)!, "prototype"));
112+
solver.addTokenConstraint(t, a.varProducer.objPropVar(specials.get(name)!, "prototype"));
113113
return t;
114114
}
115115

@@ -127,8 +127,8 @@ export function buildNatives(solver: Solver, moduleInfo: ModuleInfo): [Array<Ide
127127
if (options.natives) {
128128
const t = a.canonicalizeToken(new NativeObjectToken(`${x.name}.${f.name}`, undefined, f.invoke));
129129
if (m.name === "ecmascript")
130-
natives.set(t.name, t);
131-
solver.addTokenConstraint(t, a.varProducer.objPropVar(natives.get(x.name)!, f.name));
130+
specials.set(t.name, t);
131+
solver.addTokenConstraint(t, a.varProducer.objPropVar(specials.get(x.name)!, f.name));
132132
}
133133
}
134134

@@ -139,7 +139,7 @@ export function buildNatives(solver: Solver, moduleInfo: ModuleInfo): [Array<Ide
139139
if (options.natives) {
140140
const t = a.canonicalizeToken(new NativeObjectToken(`${x.name}.prototype.${f.name}`, undefined, f.invoke));
141141
if (m.name === "ecmascript")
142-
natives.set(t.name, t);
142+
specials.set(t.name, t);
143143
solver.addTokenConstraint(t, a.varProducer.objPropVar(pro, f.name));
144144
}
145145
}
@@ -188,10 +188,10 @@ export function buildNatives(solver: Solver, moduleInfo: ModuleInfo): [Array<Ide
188188
if (m.init) {
189189
if (logger.isVerboseEnabled())
190190
logger.verbose(`Running initialization for ${m.name}`);
191-
m.init({solver, moduleInfo, natives});
191+
m.init({solver, moduleInfo, natives: specials});
192192
}
193193
if (logger.isVerboseEnabled())
194194
logger.verbose("Adding natives completed");
195195

196-
return [globals, natives];
196+
return {globals, globalsHidden, specials};
197197
}

‎src/output/analysisstatereporter.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ export class AnalysisStateReporter {
8787
const fileIndex = fileIndices.get(fun instanceof ModuleInfo ? fun : fun.moduleInfo);
8888
if (fileIndex === undefined)
8989
assert.fail(`File index not found for ${fun}`);
90-
fs.writeSync(fd, `${first ? "" : ","}\n "${funIndex}": ${JSON.stringify(makeLocStr(fileIndex, fun.loc))}`);
90+
fs.writeSync(fd, `${first ? "" : ","}\n "${funIndex}": ${JSON.stringify(makeLocStr(fileIndex, fun.node?.loc))}`);
9191
first = false;
9292
}
9393
fs.writeSync(fd, `\n },\n "calls": {`);
@@ -176,7 +176,7 @@ export class AnalysisStateReporter {
176176
const fileIndex = fileIndices.get(fun instanceof ModuleInfo ? fun : fun.moduleInfo);
177177
if (fileIndex === undefined)
178178
assert.fail(`File index not found for ${fun}`);
179-
cg.functions[funIndex] = makeLocStr(fileIndex, fun.loc);
179+
cg.functions[funIndex] = makeLocStr(fileIndex, fun.node?.loc);
180180
}
181181
const callIndices = new Map<Node, number>();
182182
for (const [m, calls] of this.a.calls)

‎src/output/graphviz.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ export function toDot(a: AnalysisState, fd: number = process.stdout.fd) {
4141
ids.set(f, moduleId);
4242
else
4343
writeSync(fd, `${ind(i)}subgraph cluster${id(f)} {\n` +
44-
`${ind(i)} label=\"${f.name ?? "<anon>"}@${sourceLocationToString(f.loc)}\";\n` +
44+
`${ind(i)} label=\"${f.name ?? "<anon>"}@${sourceLocationToString(f.node.loc)}\";\n` +
4545
`${ind(i)} bgcolor=\"#ffffff\";\n` +
4646
`${ind(i)}node${id(f)}[style=invis,shape=point];\n`);
4747
for (const fs of f.functions.values())

‎src/output/tostringwithcode.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import {FunctionInfo} from "../analysis/infos";
1818
* Returns a string description of the given function, with a code snippet.
1919
*/
2020
export function funcToStringWithCode(info: FunctionInfo): string {
21-
return `'${codeFromLocation(info.loc)}'${info}`;
21+
return `'${codeFromLocation(info.node.loc)}'${info}`;
2222
}
2323

2424
/**

0 commit comments

Comments
 (0)