Bundle metamask compat2

packages/compartment-mapper/src/bundle-cjs.js

@@ -32,7 +32,7 @@ export default {
   getBundlerKit({
     index,
     indexedImports,
-    record: { cjsFunctor, exports: exportsList = {} },
+    record: { cjsFunctor, exports: exportsList = [] },
   }) {
     const importsMap = JSON.stringify(indexedImports);
 

packages/compartment-mapper/src/bundle-json.js

@@ -0,0 +1,22 @@
+import bundleCjs from './bundle-cjs.js';
+
+/** quotes strings */
+const q = JSON.stringify;
+
+export default {
+  runtime: bundleCjs.runtime,
+  getBundlerKit: ({
+      index,
+      indexedImports,
+      record: { jsonResult, location, exports: exportsList = [] },
+    }) => {
+      const source = `module.exports = ${q(jsonResult)}`
+      const cjsWrappedSource = `(function (_, exports) { ${source} //*/\n})\n`;
+      const cjsFunctor = `${cjsWrappedSource}//# sourceURL=${location}\n`;
+      return bundleCjs.getBundlerKit({
+        index,
+        indexedImports,
+        record: { cjsFunctor, exports: exportsList },
+    });
+  },
+}

packages/compartment-mapper/src/bundle.js

@@ -26,6 +26,7 @@ import { parseLocatedJson } from './json.js';
 
 import mjsSupport from './bundle-mjs.js';
 import cjsSupport from './bundle-cjs.js';
+import jsonSupport from './bundle-json.js';
 
 const textEncoder = new TextEncoder();
 
@@ -54,39 +55,78 @@ const sortedModules = (
   entryCompartmentName,
   entryModuleSpecifier,
 ) => {
-  const modules = [];
   const seen = new Set();
+  const modules = [];
+  const results = {};
+  const resultsPointer = {};
+
+  const keyFor = (compartmentName, moduleSpecifier) =>
+    `${compartmentName}#${moduleSpecifier}`;
 
   /**
    * @param {string} compartmentName
    * @param {string} moduleSpecifier
    */
   const recur = (compartmentName, moduleSpecifier) => {
     const key = `${compartmentName}#${moduleSpecifier}`;
-    if (seen.has(key)) {
-      return key;
+
+    // we're trying to break cycles here, but we also need the result and it might not be ready yet
+    // i tried to use a wrapper object as a reference that could be populated later,
+    // but we still try to populate resolvedImports before we have the result
+    // i guess we need to do all iteration first and then record the resolvedImports results
+    // if (seen.has(key)) {
+    //   return
+    // }
+    // // const result = {}
+    // // results.set(key, result);
+    // seen.add(key);
+    if (results[key] !== undefined) {
+      return results[key];
     }
-    seen.add(key);
+    // we need a pointer to a pointer
+    // bc we need to cycle break on the outer pointer
+    // and set the inner pointer when we have the result
+    const result = { link: { value: undefined } };
+    results[key] = result;
 
-    const resolve = compartmentResolvers[compartmentName];
     const source = compartmentSources[compartmentName][moduleSpecifier];
+    // if (['./util', './dist/backend'].includes(moduleSpecifier) && ['file:///home/xyz/Development/endo/packages/compartment-mapper/test/fixtures-0/node_modules/bundle-dep-cjs/', 'file:///home/xyz/Development/metamask-extension4/node_modules/react-devtools-core/'].includes((compartmentName))) {
+    //   console.log('recur source', moduleSpecifier, !!source)
+    // }
     if (source !== undefined) {
       const { record, parser, deferredError } = source;
+      // if (['./util', './dist/backend'].includes(moduleSpecifier) && ['file:///home/xyz/Development/endo/packages/compartment-mapper/test/fixtures-0/node_modules/bundle-dep-cjs/', 'file:///home/xyz/Development/metamask-extension4/node_modules/react-devtools-core/'].includes((compartmentName))) {
+      //   console.log('recur record', moduleSpecifier, !!record)
+      // }
       if (deferredError) {
         throw new Error(
           `Cannot bundle: encountered deferredError ${deferredError}`,
         );
       }
       if (record) {
+        const resolve = compartmentResolvers[compartmentName];
         const { imports = [], reexports = [] } =
           /** @type {PrecompiledStaticModuleInterface} */ (record);
         const resolvedImports = Object.create(null);
+        const resolvedImportsPointers = {};
         for (const importSpecifier of [...imports, ...reexports]) {
           const resolvedSpecifier = resolve(importSpecifier, moduleSpecifier);
-          resolvedImports[importSpecifier] = recur(
+          // if (['./util', './dist/backend'].includes(importSpecifier) && ['file:///home/xyz/Development/endo/packages/compartment-mapper/test/fixtures-0/node_modules/bundle-dep-cjs/', 'file:///home/xyz/Development/metamask-extension4/node_modules/react-devtools-core/'].includes((compartmentName))) {
+          //   console.log(
+          //     `initial resolved ${importSpecifier} from ${moduleSpecifier} to ${resolvedSpecifier} in ${compartmentName}`,
+          //   );
+          // }
+          // setup pointer
+          resolvedImportsPointers[importSpecifier] = recur(
             compartmentName,
             resolvedSpecifier,
           );
+          // resolvedImports[importSpecifier] = resolvedResult.value
+          // if (['./util', './dist/backend'].includes(importSpecifier) && ['file:///home/xyz/Development/endo/packages/compartment-mapper/test/fixtures-0/node_modules/bundle-dep-cjs/', 'file:///home/xyz/Development/metamask-extension4/node_modules/react-devtools-core/'].includes((compartmentName))) {
+          //   console.log(
+          //     `final resolved ${importSpecifier} from ${moduleSpecifier} to ${resolvedImports[importSpecifier]} in ${compartmentName}`,
+          //   );
+          // }
         }
 
         modules.push({
@@ -96,13 +136,19 @@ const sortedModules = (
           parser,
           record,
           resolvedImports,
+          resolvedImportsPointers,
         });
 
-        return key;
+        // results[key] = key;
+        result.link.value = key;
+        return result
       }
     } else {
       const descriptor =
         compartmentDescriptors[compartmentName].modules[moduleSpecifier];
+      // if (['./util', './dist/backend'].includes(moduleSpecifier) && ['file:///home/xyz/Development/endo/packages/compartment-mapper/test/fixtures-0/node_modules/bundle-dep-cjs/', 'file:///home/xyz/Development/metamask-extension4/node_modules/react-devtools-core/'].includes((compartmentName))) {
+      //   console.log('recur descriptor', moduleSpecifier, !!descriptor)
+      // }
       if (descriptor) {
         const {
           compartment: aliasCompartmentName,
@@ -112,7 +158,10 @@ const sortedModules = (
           aliasCompartmentName !== undefined &&
           aliasModuleSpecifier !== undefined
         ) {
-          return recur(aliasCompartmentName, aliasModuleSpecifier);
+          // resultsPointer[key] = keyFor(aliasCompartmentName, aliasModuleSpecifier);
+          result.link = recur(aliasCompartmentName, aliasModuleSpecifier).link;
+          // TODO: test if value is ever undefined, if not we may not need the inner pointer
+          return result
         }
       }
     }
@@ -122,14 +171,50 @@ const sortedModules = (
     );
   };
 
+  // walk graph
   recur(entryCompartmentName, entryModuleSpecifier);
+  // console.log('results', (Object.values(results)))
+  // // finalize key pointers
+  // // reverse didnt fix the problem
+  // // copy from source to dest
+  // // hint is that "results" before here is just {key: key},
+  // // so we likely dont need it at all
+  // // i think we can just record the redirects and if there is not one,
+  // // then we can just use the key
+  // // the current setup may require recursive redirects
+  // // but not sure why we would need that --
+  // // maybe bc package name specifier (xyz) -> package main (./index) -> resolved (./index.js) 
+  // Object.entries(resultsPointer).reverse().forEach(([destKey, sourceKey]) => {
+  //   const finalKey = results[sourceKey];
+  //   // const finalKey = results[sourceKey] || sourceKey;
+  //   if (finalKey === undefined) {
+  //     throw new Error(`Cannot bundle: cannot follow pointer for ${destKey} from ${sourceKey}`);
+  //   }
+  //   console.log('key pointers', destKey, sourceKey)
+  //   results[destKey] = finalKey;
+  // });
+  // finalize resolvedImports pointers
+  // Object.values(modules).forEach(({ key, resolvedImports, resolvedImportsPointers }) => {
+  //   Object.entries(resolvedImportsPointers).forEach(([importSpecifier, destKey]) => {
+  //     const finalKey = results[destKey];
+  //     console.log('resolvedImports', key, importSpecifier, finalKey)
+  //     resolvedImports[importSpecifier] = finalKey;
+  //   });
+  // })
+  modules.forEach(({ key, resolvedImports, resolvedImportsPointers }) => {
+    Object.entries(resolvedImportsPointers).forEach(([importSpecifier, { link: { value } }]) => {
+      // console.log('resolvedImports', importSpecifier, value)
+      resolvedImports[importSpecifier] = value;
+    });
+  })
 
   return modules;
 };
 
 const implementationPerParser = {
   'pre-mjs-json': mjsSupport,
   'pre-cjs-json': cjsSupport,
+  'json': jsonSupport,
 };
 
 function getRuntime(parser) {
@@ -143,10 +228,11 @@ function getBundlerKitForModule(module) {
   if (!implementationPerParser[parser]) {
     const warning = `/*unknown parser:${parser}*/`;
     // each item is a function to avoid creating more in-memory copies of the source text etc.
+    throw new Error(`cannot bundle: unknown parser: ${parser} for module ${module.key}`)
     return {
       getFunctor: () => `(()=>{${warning}})`,
-      getCells: `{${warning}}`,
-      getFunctorCall: warning,
+      getCells: () => `{${warning}}`,
+      getFunctorCall: () => warning,
     };
   }
   const getBundlerKit = implementationPerParser[parser].getBundlerKit;
@@ -231,18 +317,28 @@ export const makeBundle = async (read, moduleLocation, options) => {
   // Create an index of modules so we can resolve import specifiers to the
   // index of the corresponding functor.
   const modulesByKey = Object.create(null);
-  for (let index = 0; index < modules.length; index += 1) {
-    const module = modules[index];
-    module.index = index;
+  let moduleIndex = 0;
+  for (const module of modules) {
+    module.index = moduleIndex;
     modulesByKey[module.key] = module;
+    moduleIndex++
   }
   const parsersInUse = new Set();
   for (const module of modules) {
+    // console.log('resolvedImports', module.resolvedImports)
     module.indexedImports = Object.fromEntries(
-      Object.entries(module.resolvedImports).map(([importSpecifier, key]) => [
-        importSpecifier,
-        modulesByKey[key].index,
-      ]),
+      // specifiers completed by candidates cant be bundled?
+      Object.entries(module.resolvedImports).map(([importSpecifier, key]) => {
+        if (modulesByKey[key] === undefined) {
+          throw new Error(
+            `Cannot bundle: cannot find module ${key} for import ${importSpecifier} in module ${module.key} (with resolvedImports ${Object.keys(module.resolvedImports)}`,
+          );
+        }
+        return [
+          importSpecifier,
+          modulesByKey[key].index,
+        ]
+      }),
     );
     parsersInUse.add(module.parser);
     module.bundlerKit = getBundlerKitForModule(module);
@@ -281,7 +377,7 @@ ${''.concat(...modules.map(m => m.bundlerKit.getCells()))}\
 
 ${''.concat(...modules.map(m => m.bundlerKit.getReexportsWiring()))}\
 
-  const namespaces = cells.map(cells => Object.freeze(Object.create(null, cells)));
+  const namespaces = cells.map((cell, index) => Object.freeze(Object.create(null, cell)));
 
   for (let index = 0; index < namespaces.length; index += 1) {
     cells[index]['*'] = cell('*', namespaces[index]);

packages/compartment-mapper/src/import-hook.js

@@ -131,6 +131,7 @@ export const makeImportHookMaker = (
         deferredError: error.message,
       };
 
+      throw error
       return record;
     };
 

packages/compartment-mapper/src/infer-exports.js

@@ -168,7 +168,7 @@ export const inferExportsAndAliases = (
   // expose default module as package root
   // may be overwritten by browser field
   // see https://github.com/endojs/endo/issues/1363
-  if (module === undefined && exports === undefined) {
+  if (module === undefined) {
     const defaultModule = main !== undefined ? relativize(main) : './index.js';
     externalAliases['.'] = defaultModule;
     // in commonjs, expose package root as default module

packages/compartment-mapper/src/node-modules.js

@@ -36,7 +36,7 @@
  */
 
 /**
- * @typedef {Record<string, {spec: string, alias: string}>} CommonDependencyDescriptors
+ * @typedef {Record<string, {spec: string, name: string}>} CommonDependencyDescriptors
  */
 
 import { inferExportsAndAliases } from './infer-exports.js';
@@ -276,8 +276,8 @@ const graphPackage = async (
     devDependencies = {},
   } = packageDescriptor;
   const allDependencies = {};
-  assign(allDependencies, commonDependencyDescriptors);
-  for (const [name, { spec }] of Object.entries(commonDependencyDescriptors)) {
+  // assign(allDependencies, commonDependencyDescriptors);
+  for (const { spec, name } of Object.values(commonDependencyDescriptors)) {
     allDependencies[name] = spec;
   }
   assign(allDependencies, dependencies);
@@ -363,8 +363,13 @@ const graphPackage = async (
 
   await Promise.all(children);
 
+  // console.log(name)
+  // if (name === 'call-bind') {
+  //   console.log(result, packageDescriptor)
+  // }
+
   // handle commonDependencyDescriptors package aliases
-  for (const [name, { alias }] of Object.entries(commonDependencyDescriptors)) {
+  for (const [alias, { name }] of Object.entries(commonDependencyDescriptors)) {
     // update the dependencyLocations to point to the common dependency
     const targetLocation = dependencyLocations[name];
     if (targetLocation === undefined) {
@@ -374,6 +379,9 @@ const graphPackage = async (
     }
     dependencyLocations[alias] = targetLocation;
   }
+  // if (name === 'pump') {
+  //   console.log('commonDependencyDescriptors set', result.label, dependencyLocations, commonDependencyDescriptors)
+  // }
   // handle internalAliases package aliases
   for (const specifier of keys(internalAliases).sort()) {
     const target = internalAliases[specifier];
@@ -508,9 +516,9 @@ const graphPackages = async (
         `Cannot find dependency ${dependencyName} for ${packageLocation} from common dependencies`,
       );
     }
-    commonDependencyDescriptors[dependencyName] = {
+    commonDependencyDescriptors[alias] = {
       spec,
-      alias,
+      name: dependencyName,
     };
   }
 

packages/compartment-mapper/src/parse-json.js

@@ -21,21 +21,25 @@ export const parseJson = async (
 ) => {
   const source = textDecoder.decode(bytes);
   const imports = freeze([]);
-
+  const jsonResult = parseLocatedJson(source, location);
   /**
    * @param {Object} exports
    */
   const execute = exports => {
-    exports.default = parseLocatedJson(source, location);
+    exports.default = jsonResult;
   };
+  const record = freeze({
+    imports,
+    exports: freeze(['default']),
+    execute: freeze(execute),
+    jsonResult,
+    location,
+  })
+
   return {
     parser: 'json',
     bytes,
-    record: freeze({
-      imports,
-      exports: freeze(['default']),
-      execute: freeze(execute),
-    }),
+    record,
   };
 };