diff --git a/packages/compartment-mapper/NEWS.md b/packages/compartment-mapper/NEWS.md index 1a749a4d1e..ed45043cfd 100644 --- a/packages/compartment-mapper/NEWS.md +++ b/packages/compartment-mapper/NEWS.md @@ -23,6 +23,27 @@ User-visible changes to `@endo/compartment-mapper`: originally intended. To those who expected the previous behavior: if you exist, please exercise caution when upgrading. +Experimental: + +- The module `@endo/compartment-mapper/import-archive-parsers.js` does not + support modules in archives in their original ESM (`mjs`) or CommonJS (`cjs`) + formats because they entrain Babel and a full JavaScript lexer that are + not suitable for use in all environments, specifically XS. + This version introduces an elective + `@endo/compartment-mapper/import-archive-all-parsers.js` that has all of the + precompiled module parsers (`pre-cjs-json` and `pre-mjs-json`) that Endo's + bundler currently produces by default and additionally parsers for original + sources (`mjs`, `cjs`). + Also, provided the `xs` package condition, + `@endo/compartment-mapper/import-archive-parsers.js` now falls through to the + native `ModuleSource` and safely includes `mjs` and `cjs` without entraining + Babel, but is only supported in conjunction with the `__native__` option + for `Compartment`, `importArchive`, `parseArchive`, and `importBundle`. + With the `node` package condition (present by default when running ESM on + `node`), `@endo/compartment-mapper/import-archive-parsers.js` also now + includes `mjs` and `cjs` by entraining Babel, which performs adequately on + that platform. + # v1.3.0 (2024-10-10) - Adds support for dynamic requires in CommonJS modules. This requires specific @@ -30,6 +51,15 @@ User-visible changes to `@endo/compartment-mapper`: enabled by default. See the signature of `loadFromMap()` in `import-lite.js` for details. +Experimental: + +- Adds a `__native__: true` option to all paths to import, that indicates that + the application will fall through to the native implementation of + Compartment, currently only available on XS, which lacks support for + precompiled module sources (as exist in many archived applications, + particularly Agoric smart contract bundles) and instead supports loading + modules from original sources (which is not possible at runtime on XS). + # v1.2.0 (2024-07-30) - Fixes incompatible behavior with Node.js package conditional exports #2276. diff --git a/packages/compartment-mapper/README.md b/packages/compartment-mapper/README.md index 6cfbdfb6b9..7a7fd0ad19 100644 --- a/packages/compartment-mapper/README.md +++ b/packages/compartment-mapper/README.md @@ -328,6 +328,20 @@ These will be appended to each module from the archive, for debugging purposes. The `@endo/bundle-source` and `@endo/import-bundle` tools integrate source maps for an end-to-end debugging experience. +# XS (experimental) + +The Compartment Mapper can use native XS `Compartment` and `ModuleSource` under +certain conditions: + +1. The application must be an XS script that was compiled with the `xs` + package condition. + This causes `ses`, `@endo/module-source`, and `@endo/import-bundle` to + provide slightly different implementations that can fall through to native + behavior. +2. The application must opt-in with the `__native__: true` option on any + of the compartment mapper methods that import modules like `importLocation` + and `importArchive`. + # Design Each of the workflows the compartment mapper executes a portion of one sequence diff --git a/packages/compartment-mapper/import-archive-all-parsers.js b/packages/compartment-mapper/import-archive-all-parsers.js new file mode 100644 index 0000000000..f31eb40d75 --- /dev/null +++ b/packages/compartment-mapper/import-archive-all-parsers.js @@ -0,0 +1 @@ +export { defaultParserForLanguage } from './src/import-archive-all-parsers.js'; diff --git a/packages/compartment-mapper/package.json b/packages/compartment-mapper/package.json index 3cb623d0b6..60de4e8416 100644 --- a/packages/compartment-mapper/package.json +++ b/packages/compartment-mapper/package.json @@ -32,7 +32,12 @@ "./capture-lite.js": "./capture-lite.js", "./import-archive.js": "./import-archive.js", "./import-archive-lite.js": "./import-archive-lite.js", - "./import-archive-parsers.js": "./import-archive-parsers.js", + "./import-archive-parsers.js": { + "xs": "./import-archive-all-parsers.js", + "node": "./import-archive-all-parsers.js", + "default": "./import-archive-parsers.js" + }, + "./import-archive-all-parsers.js": "./import-archive-all-parsers.js", "./bundle.js": "./bundle.js", "./node-powers.js": "./node-powers.js", "./node-modules.js": "./node-modules.js", diff --git a/packages/compartment-mapper/src/bundle.js b/packages/compartment-mapper/src/bundle.js index fe63135f1b..bea4f6a657 100644 --- a/packages/compartment-mapper/src/bundle.js +++ b/packages/compartment-mapper/src/bundle.js @@ -117,8 +117,14 @@ const sortedModules = ( const source = compartmentSources[compartmentName][moduleSpecifier]; if (source !== undefined) { const { record, parser, deferredError, bytes } = source; - assert(parser !== undefined); - assert(bytes !== undefined); + assert( + bytes !== undefined, + `No bytes for ${moduleSpecifier} in ${compartmentName}`, + ); + assert( + parser !== undefined, + `No parser for ${moduleSpecifier} in ${compartmentName}`, + ); if (deferredError) { throw Error( `Cannot bundle: encountered deferredError ${deferredError}`, diff --git a/packages/compartment-mapper/src/import-archive-all-parsers.js b/packages/compartment-mapper/src/import-archive-all-parsers.js new file mode 100644 index 0000000000..336084d007 --- /dev/null +++ b/packages/compartment-mapper/src/import-archive-all-parsers.js @@ -0,0 +1,29 @@ +/* Provides a set of default language behaviors (parsers) suitable for + * evaluating archives (zip files with a `compartment-map.json` and a file for + * each module) with pre-compiled _or_ original ESM and CommonJS. + * + * This module does not entrain a dependency on Babel on XS, but does on other + * platforms like Node.js. + */ +/** @import {ParserForLanguage} from './types.js' */ + +import parserPreCjs from './parse-pre-cjs.js'; +import parserJson from './parse-json.js'; +import parserText from './parse-text.js'; +import parserBytes from './parse-bytes.js'; +import parserPreMjs from './parse-pre-mjs.js'; +import parserMjs from './parse-mjs.js'; +import parserCjs from './parse-cjs.js'; + +/** @satisfies {Readonly} */ +export const defaultParserForLanguage = Object.freeze( + /** @type {const} */ ({ + 'pre-cjs-json': parserPreCjs, + 'pre-mjs-json': parserPreMjs, + cjs: parserCjs, + mjs: parserMjs, + json: parserJson, + text: parserText, + bytes: parserBytes, + }), +); diff --git a/packages/compartment-mapper/src/import-archive-lite.js b/packages/compartment-mapper/src/import-archive-lite.js index 5c50c41358..a0728c6dcc 100644 --- a/packages/compartment-mapper/src/import-archive-lite.js +++ b/packages/compartment-mapper/src/import-archive-lite.js @@ -257,6 +257,7 @@ export const parseArchive = async ( modules = undefined, importHook: exitModuleImportHook = undefined, parserForLanguage: parserForLanguageOption = {}, + __native__ = false, } = options; const parserForLanguage = freeze( @@ -343,6 +344,7 @@ export const parseArchive = async ( }), ), Compartment: CompartmentParseOption, + __native__, }); await pendingJobsPromise; @@ -362,6 +364,7 @@ export const parseArchive = async ( transforms, __shimTransforms__, Compartment: CompartmentOption = CompartmentParseOption, + __native__, importHook: exitModuleImportHook, } = options || {}; @@ -388,6 +391,7 @@ export const parseArchive = async ( transforms, __shimTransforms__, Compartment: CompartmentOption, + __native__, }); await pendingJobsPromise; diff --git a/packages/compartment-mapper/src/import-lite.js b/packages/compartment-mapper/src/import-lite.js index 71d748f992..51c4a0e06b 100644 --- a/packages/compartment-mapper/src/import-lite.js +++ b/packages/compartment-mapper/src/import-lite.js @@ -152,6 +152,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { transforms, __shimTransforms__, Compartment: CompartmentOption = LoadCompartmentOption, + __native__, importHook: exitModuleImportHook, } = options; const compartmentExitModuleImportHook = exitModuleImportHookMaker({ @@ -201,6 +202,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { syncModuleTransforms, __shimTransforms__, Compartment: CompartmentOption, + __native__, })); } else { // sync module transforms are allowed, because they are "compatible" @@ -215,6 +217,7 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { syncModuleTransforms, __shimTransforms__, Compartment: CompartmentOption, + __native__, })); } diff --git a/packages/compartment-mapper/src/link.js b/packages/compartment-mapper/src/link.js index b2b1414f5d..b2f70f9652 100644 --- a/packages/compartment-mapper/src/link.js +++ b/packages/compartment-mapper/src/link.js @@ -258,6 +258,7 @@ export const link = ( moduleTransforms, syncModuleTransforms, __shimTransforms__ = [], + __native__ = false, archiveOnly = false, Compartment = defaultCompartment, } = options; @@ -362,6 +363,7 @@ export const link = ( transforms, __shimTransforms__, __options__: true, + __native__, }); if (!archiveOnly) { diff --git a/packages/compartment-mapper/src/types/external.ts b/packages/compartment-mapper/src/types/external.ts index d0de4e9916..6a6cd95d81 100644 --- a/packages/compartment-mapper/src/types/external.ts +++ b/packages/compartment-mapper/src/types/external.ts @@ -29,6 +29,7 @@ export type ExecuteOptions = Partial<{ __shimTransforms__: Array; attenuations: Record; Compartment: typeof Compartment; + __native__: boolean; }> & ModulesOption & ExitModuleImportHookOption; @@ -38,6 +39,7 @@ export type ParseArchiveOptions = Partial<{ computeSha512: HashFn; computeSourceLocation: ComputeSourceLocationHook; computeSourceMapLocation: ComputeSourceMapLocationHook; + __native__: boolean; }> & ModulesOption & CompartmentOption & diff --git a/packages/compartment-mapper/src/types/internal.ts b/packages/compartment-mapper/src/types/internal.ts index 8cce0cd5f4..bb6c6c26f3 100644 --- a/packages/compartment-mapper/src/types/internal.ts +++ b/packages/compartment-mapper/src/types/internal.ts @@ -44,6 +44,7 @@ export type LinkOptions = { moduleTransforms?: ModuleTransforms; syncModuleTransforms?: SyncModuleTransforms; archiveOnly?: boolean; + __native__?: boolean; } & ExecuteOptions; export type LinkResult = { diff --git a/packages/ses/.gitignore b/packages/ses/.gitignore new file mode 100644 index 0000000000..a9a5aecf42 --- /dev/null +++ b/packages/ses/.gitignore @@ -0,0 +1 @@ +tmp diff --git a/packages/ses/NEWS.md b/packages/ses/NEWS.md index 4bb7cba98e..2818fed0e6 100644 --- a/packages/ses/NEWS.md +++ b/packages/ses/NEWS.md @@ -1,5 +1,17 @@ User-visible changes in `ses`: +# Next release + +Incubating: Please do not rely on these features as they are under development +and subject to breaking changes that will not be signaled by semver. + +- Adds support for an XS-specific variant of the SES shim that is triggered + with the `xs` package export condition. + This version of SES preserves all the features of `Compartment` provided + uniquely by the SES shim, but with the `__native__` constructor option, + loses support for importing precompiled module records and gains support + for native `ModuleSource`. + # v1.10.0 (2024-11-13) - Permit [Promise.try](https://github.com/tc39/proposal-promise-try), diff --git a/packages/ses/package.json b/packages/ses/package.json index c6e77c27af..7d0a735ed2 100644 --- a/packages/ses/package.json +++ b/packages/ses/package.json @@ -38,6 +38,7 @@ ".": { "import": { "types": "./types.d.ts", + "xs": "./src-xs/index.js", "default": "./index.js" }, "require": { @@ -57,8 +58,14 @@ }, "./tools.js": "./tools.js", "./assert-shim.js": "./assert-shim.js", - "./lockdown-shim.js": "./lockdown-shim.js", - "./compartment-shim.js": "./compartment-shim.js", + "./lockdown-shim.js": { + "xs": "./src-xs/lockdown-shim.js", + "default": "./lockdown-shim.js" + }, + "./compartment-shim.js": { + "xs": "./src-xs/compartment-shim.js", + "default": "./compartment-shim.js" + }, "./console-shim.js": "./console-shim.js", "./package.json": "./package.json" }, @@ -74,7 +81,7 @@ "prepare": "npm run clean && npm run build", "qt": "ava", "test": "tsd && ava", - "test:xs": "xst dist/ses.umd.js test/_lockdown-safe.js", + "test:xs": "xst dist/ses.umd.js test/_lockdown-safe.js && node scripts/generate-test-xs.js && xst tmp/test-xs.js", "postpack": "git clean -f '*.d.ts*' '*.tsbuildinfo'" }, "dependencies": { diff --git a/packages/ses/scripts/generate-test-xs.js b/packages/ses/scripts/generate-test-xs.js new file mode 100644 index 0000000000..79588a9fcb --- /dev/null +++ b/packages/ses/scripts/generate-test-xs.js @@ -0,0 +1,56 @@ +/* eslint-env node */ +/* glimport/no-extraneous-dependenciesobal process */ +import '../index.js'; +import { promises as fs } from 'fs'; +// Lerna does not like dependency cycles. +// With an explicit devDependency from module-source to compartment-mapper, +// the build script stalls before running every package's build script. +// yarn lerna run build +// Omitting the dependency from package.json solves the problem and works +// by dint of shared workspace node_modules. +// eslint-disable-next-line import/no-extraneous-dependencies +import { makeBundle } from '@endo/compartment-mapper/bundle.js'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { ModuleSource } from '@endo/module-source'; +import { fileURLToPath } from 'url'; + +const read = async location => { + const path = fileURLToPath(location); + return fs.readFile(path); +}; +const write = async (location, content) => { + const path = fileURLToPath(location); + await fs.writeFile(path, content); +}; + +const main = async () => { + await fs.mkdir(fileURLToPath(new URL('../tmp', import.meta.url)), { + recursive: true, + }); + + const meaningText = await fs.readFile( + fileURLToPath(new URL('../test/_meaning.js', import.meta.url)), + 'utf8', + ); + const meaningModuleSource = new ModuleSource(meaningText); + + await fs.writeFile( + fileURLToPath(new URL('../tmp/_meaning.pre-mjs.json', import.meta.url)), + JSON.stringify(meaningModuleSource), + ); + + const xsPrelude = await makeBundle( + read, + new URL('../test/_xs.js', import.meta.url).href, + { + tags: new Set(['xs']), + }, + ); + + await write(new URL('../tmp/test-xs.js', import.meta.url).href, xsPrelude); +}; + +main().catch(err => { + console.error('Error running main:', err); + process.exitCode = 1; +}); diff --git a/packages/ses/src-xs/commons.js b/packages/ses/src-xs/commons.js new file mode 100644 index 0000000000..ca9cc5c2f5 --- /dev/null +++ b/packages/ses/src-xs/commons.js @@ -0,0 +1,22 @@ +/** + * @module In the spirit of ../src/commons.js, this module captures native + * functions specific to the XS engine during initialization, so vetted shims + * are free to modify any intrinsic without risking the integrity of SES. + */ + +import { + getOwnPropertyDescriptor, + globalThis, + uncurryThis, +} from '../src/commons.js'; + +export const NativeStartCompartment = globalThis.Compartment; +export const nativeCompartmentPrototype = NativeStartCompartment.prototype; +export const nativeImport = uncurryThis(nativeCompartmentPrototype.import); +export const nativeImportNow = uncurryThis( + nativeCompartmentPrototype.importNow, +); +export const nativeEvaluate = uncurryThis(nativeCompartmentPrototype.evaluate); +export const nativeGetGlobalThis = uncurryThis( + getOwnPropertyDescriptor(nativeCompartmentPrototype, 'globalThis').get, +); diff --git a/packages/ses/src-xs/compartment-shim.js b/packages/ses/src-xs/compartment-shim.js new file mode 100644 index 0000000000..d250e07047 --- /dev/null +++ b/packages/ses/src-xs/compartment-shim.js @@ -0,0 +1,128 @@ +/** + * @module Provides a XS-specific variation on the behavior of + * ../compartment-shim.js, completing the story that begins in + * ./compartment.js, adding a Compartment constructor adapter to the global + * scope and transforming all of the methods of a native compartment into + * thunks that will alternately delegate to its native or shim behaviors + * depending on the __native__ Compartment constructor option. + */ + +import { defineProperty, globalThis, weakmapGet } from '../src/commons.js'; +import { + NativeStartCompartment, + nativeCompartmentPrototype, + nativeImport, + nativeImportNow, + nativeEvaluate, + nativeGetGlobalThis, +} from './commons.js'; +import { + ShimStartCompartment, + adaptCompartmentConstructors, + privateFields, + shimEvaluate, + shimGetGlobalThis, + shimImport, + shimImportNow, +} from './compartment.js'; + +const adapterFunctions = { + evaluate(source, options) { + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return nativeEvaluate(source); + } + const { transforms, delegateNative, nativeEval } = fields; + for (const transform of transforms) { + source = transform(source); + } + if (delegateNative) { + return nativeEval(source); + } else { + return shimEvaluate(source, options); + } + }, + + async import(specifier) { + await null; + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return nativeImport(this, specifier); + } + const { noNamespaceBox, delegateNative } = fields; + const delegateImport = delegateNative ? nativeImport : shimImport; + const namespace = delegateImport(this, specifier); + return noNamespaceBox ? namespace : { namespace: await namespace }; + }, + + importNow(specifier) { + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return nativeImportNow(this, specifier); + } + const { delegateNative } = fields; + const delegateImportNow = delegateNative ? nativeImportNow : shimImportNow; + return delegateImportNow(this, specifier); + }, +}; + +defineProperty(nativeCompartmentPrototype, 'evaluate', { + value: adapterFunctions.evaluate, + writable: true, + configurable: true, + enumerable: false, +}); + +defineProperty(nativeCompartmentPrototype, 'import', { + value: adapterFunctions.import, + writable: true, + configurable: true, + enumerable: false, +}); + +defineProperty(nativeCompartmentPrototype, 'importNow', { + value: adapterFunctions.importNow, + writable: true, + configurable: true, + enumerable: false, +}); + +defineProperty(nativeCompartmentPrototype, 'globalThis', { + get() { + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return nativeGetGlobalThis(this); + } + const { delegateNative } = fields; + const delegateGetGlobalThis = delegateNative + ? nativeGetGlobalThis + : shimGetGlobalThis; + return delegateGetGlobalThis(this); + }, + configurable: true, + enumerable: false, +}); + +defineProperty(nativeCompartmentPrototype, 'name', { + get() { + const fields = weakmapGet(privateFields, this); + if (fields === undefined) { + return undefined; + } + const { name } = fields; + return name; + }, + configurable: true, + enumerable: false, +}); + +// Adapt the start compartment's native Compartment to the SES-compatibility +// adapter. +// Before Lockdown, the Compartment constructor in transitive child +// Compartments is not (and cannot be) hardened. +const noHarden = object => object; +globalThis.Compartment = adaptCompartmentConstructors( + NativeStartCompartment, + ShimStartCompartment, + noHarden, +); diff --git a/packages/ses/src-xs/compartment.js b/packages/ses/src-xs/compartment.js new file mode 100644 index 0000000000..f8d3a791ba --- /dev/null +++ b/packages/ses/src-xs/compartment.js @@ -0,0 +1,284 @@ +/* eslint-disable @endo/no-nullish-coalescing */ +import { + Map, + Object, + SyntaxError, + TypeError, + WeakMap, + arrayMap, + create, + defineProperty, + entries, + fromEntries, + getOwnPropertyDescriptor, + globalThis, + mapDelete, + mapGet, + mapSet, + uncurryThis, + weakmapGet, + weakmapSet, +} from '../src/commons.js'; +import { nativeGetGlobalThis } from './commons.js'; +import { + makeCompartmentConstructor, + compartmentOptions, +} from '../src/compartment.js'; +import { getGlobalIntrinsics } from '../src/intrinsics.js'; +import { tameFunctionToString } from '../src/tame-function-tostring.js'; +import { chooseReporter } from '../src/reporting.js'; + +const muteReporter = chooseReporter('none'); + +export const ShimStartCompartment = makeCompartmentConstructor( + makeCompartmentConstructor, + getGlobalIntrinsics(globalThis, muteReporter), + tameFunctionToString(), +); + +export const shimCompartmentPrototype = ShimStartCompartment.prototype; + +export const shimEvaluate = uncurryThis(shimCompartmentPrototype.evaluate); +export const shimImport = uncurryThis(shimCompartmentPrototype.import); +export const shimImportNow = uncurryThis(shimCompartmentPrototype.importNow); + +export const shimGetGlobalThis = uncurryThis( + getOwnPropertyDescriptor(ShimStartCompartment.prototype, 'globalThis').get, +); + +export const privateFields = new WeakMap(); + +const adaptVirtualModuleSource = ({ + execute, + imports = [], + exports = [], + reexports = [], +}) => { + const resolutions = create(null); + let i = 0; + return { + execute(environment) { + const fakeCompartment = { + importNow(specifier) { + return environment[specifier]; + }, + }; + execute(environment, fakeCompartment, resolutions); + }, + bindings: [ + ...arrayMap(imports, specifier => { + const resolved = `_${i}`; + i += 1; + resolutions[specifier] = resolved; + return { importAllFrom: specifier, as: resolved }; + }), + ...arrayMap(reexports, specifier => ({ exportAllFrom: specifier })), + ...arrayMap(exports, name => ({ export: name })), + ], + }; +}; + +const adaptModuleSource = source => { + if (source.execute) { + return adaptVirtualModuleSource(source); + } + // eslint-disable-next-line no-underscore-dangle + if (source.__syncModuleProgram__) { + throw new SyntaxError( + 'XS native compartments do not support precompiled module sources', + ); + } + return source; +}; + +const adaptModuleDescriptor = ( + descriptor, + specifier, + compartment = undefined, +) => { + if (Object(descriptor) !== descriptor) { + throw new TypeError('module descriptor must be an object'); + } + if (descriptor.namespace !== undefined) { + return descriptor; + } + if (descriptor.source !== undefined) { + return { + source: adaptModuleSource(descriptor.source), + importMeta: descriptor.importMeta, + specifier: descriptor.specifier, + }; + } + // Legacy support for record descriptors. + if (descriptor.record !== undefined) { + if ( + descriptor.specifier === specifier || + descriptor.specifier === undefined + ) { + return { + source: adaptModuleSource(descriptor.record), + specifier, + importMeta: descriptor.importMeta, + }; + } else { + if (compartment === undefined) { + throw new TypeError( + 'Cannot construct forward reference module descriptor in module map', + ); + } + const { descriptors } = weakmapGet(privateFields, compartment); + mapSet(descriptors, descriptor.specifier, { + compartment, + namespace: specifier, + }); + return { + source: adaptModuleSource(descriptor.record), + specifier: descriptor.specifier, + importMeta: descriptor.importMeta, + }; + } + } + if (descriptor.specifier !== undefined) { + return { + namespace: descriptor.specifier, + compartment: descriptor.compartment, + }; + } + // Legacy support for a source in the place of a descriptor. + return { source: adaptModuleSource(descriptor) }; +}; + +export const adaptCompartmentConstructors = ( + NativeCompartment, + ShimCompartment, + maybeHarden, +) => { + function Compartment(...args) { + const options = compartmentOptions(...args); + + const { + name = undefined, + globals = {}, + transforms = [], + resolveHook = () => { + throw new TypeError('Compartment requires a resolveHook'); + }, + loadHook = undefined, + loadNowHook = undefined, + importHook = loadHook, + importNowHook = loadNowHook, + moduleMapHook = () => {}, + __native__: delegateNative = false, + __noNamespaceBox__: noNamespaceBox = false, + } = options; + + const modules = delegateNative + ? fromEntries( + arrayMap( + entries(options.modules ?? {}), + ([specifier, descriptor]) => [ + specifier, + adaptModuleDescriptor(descriptor, specifier, undefined), + ], + ), + ) + : {}; + + // side table for references from one descriptor to another + const descriptors = new Map(); + + let nativeOptions = { globals, modules }; + + if (importHook) { + /** @param {string} specifier */ + const nativeImportHook = async specifier => { + await null; + let descriptor = + mapGet(descriptors, specifier) ?? + moduleMapHook(specifier) ?? + (await importHook(specifier)); + mapDelete(descriptors, specifier); + // eslint-disable-next-line no-use-before-define + descriptor = adaptModuleDescriptor(descriptor, specifier, compartment); + return descriptor; + }; + nativeOptions = { + ...nativeOptions, + resolveHook, + importHook: nativeImportHook, + loadHook: nativeImportHook, + }; + } + + if (importNowHook) { + /** @param {string} specifier */ + const nativeImportNowHook = specifier => { + let descriptor = + mapGet(descriptors, specifier) ?? + moduleMapHook(specifier) ?? + importNowHook(specifier); + mapDelete(descriptors, specifier); + // eslint-disable-next-line no-use-before-define + descriptor = adaptModuleDescriptor(descriptor, specifier, compartment); + return descriptor; + }; + nativeOptions = { + ...nativeOptions, + resolveHook, + importNowHook: nativeImportNowHook, + loadNowHook: nativeImportNowHook, + }; + } + + const compartment = new NativeCompartment(nativeOptions); + + const nativeGlobalThis = nativeGetGlobalThis(compartment); + + const nativeEval = nativeGlobalThis.eval; + + weakmapSet(privateFields, compartment, { + name, + transforms, + delegateNative, + noNamespaceBox, + nativeEval, + descriptors, + }); + + const shimOptions = { + ...options, + __noNamespaceBox__: true, + __options__: true, + }; + + uncurryThis(ShimCompartment)(compartment, shimOptions); + + const shimGlobalThis = shimGetGlobalThis(compartment); + + const ChildCompartment = adaptCompartmentConstructors( + nativeGlobalThis.Compartment, + shimGlobalThis.Compartment, + maybeHarden, + ); + + defineProperty(nativeGlobalThis, 'Compartment', { + value: ChildCompartment, + writable: true, + configurable: true, + enumerable: false, + }); + + defineProperty(shimGlobalThis, 'Compartment', { + value: ChildCompartment, + writable: true, + configurable: true, + enumerable: false, + }); + + maybeHarden(compartment); + return compartment; + } + + maybeHarden(Compartment); + return Compartment; +}; diff --git a/packages/ses/src-xs/index.js b/packages/ses/src-xs/index.js new file mode 100644 index 0000000000..eaadafa782 --- /dev/null +++ b/packages/ses/src-xs/index.js @@ -0,0 +1,26 @@ +/** + * @module This is an alternate implementation of ../index.js to provide + * access to the native implementation of Hardened JavaScript on the XS + * engine, but adapted for backward compatibility with SES. + * This module can only be reached in the presence of the package export/import + * condition "xs", and should only be used to bundle an XS-specific version of + * SES. + */ +// @ts-nocheck +/// + +import { Object, freeze } from '../src/commons.js'; + +// These are the constituent shims in an arbitrary order, but matched +// to ../index.js to remove doubt. +import './lockdown-shim.js'; +import './compartment-shim.js'; +import '../src/assert-shim.js'; +import '../src/console-shim.js'; + +// XS Object.freeze takes a second argument to apply freeze transitively, but +// with slightly different effects than `harden`. +// We disable this behavior to encourage use of `harden` for portable Hardened +// JavaScript. +/** @param {object} object */ +Object.freeze = object => freeze(object); diff --git a/packages/ses/src-xs/lockdown-shim.js b/packages/ses/src-xs/lockdown-shim.js new file mode 100644 index 0000000000..50632b7275 --- /dev/null +++ b/packages/ses/src-xs/lockdown-shim.js @@ -0,0 +1,26 @@ +/** + * @module Alters the XS implementation of Lockdown to be backward compatible + * with SES, providing Compartment constructors in every Compartment that can + * be used with either native ModuleSources or module sources pre-compiled for + * the SES Compartment, depending on the __native__ Compartment constructor + * option. + */ +import { globalThis } from '../src/commons.js'; +import { NativeStartCompartment } from './commons.js'; +import { repairIntrinsics } from '../src/lockdown.js'; +import { + ShimStartCompartment, + adaptCompartmentConstructors, +} from './compartment.js'; + +globalThis.lockdown = options => { + const hardenIntrinsics = repairIntrinsics(options); + hardenIntrinsics(); + // Replace global Compartment with a version that is hardened and hardens + // transitive child Compartment. + globalThis.Compartment = adaptCompartmentConstructors( + NativeStartCompartment, + ShimStartCompartment, + harden, + ); +}; diff --git a/packages/ses/src/commons.js b/packages/ses/src/commons.js index b4f8f6f391..5a7cb99b16 100644 --- a/packages/ses/src/commons.js +++ b/packages/ses/src/commons.js @@ -1,17 +1,19 @@ -/* global globalThis */ -/* eslint-disable no-restricted-globals */ - /** - * commons.js - * Declare shorthand functions. Sharing these declarations across modules - * improves on consistency and minification. Unused declarations are - * dropped by the tree shaking process. + * @module Captures native intrinsics during initialization, so vetted shims + * (running between initialization of SES and calling lockdown) are free to + * modify the environment without compromising the integrity of SES. For + * example, a vetted shim can modify Object.assign because we capture and + * export Object and assign here, then never again consult Object to get its + * assign property. * - * We capture these, not just for brevity, but for security. If any code - * modifies Object to change what 'assign' points to, the Compartment shim - * would be corrupted. + * This pattern of use is enforced by eslint rules no-restricted-globals and + * no-polymorphic-call. + * We maintain the list of restricted globals in ../package.json. */ +/* global globalThis */ +/* eslint-disable no-restricted-globals */ + // We cannot use globalThis as the local name since it would capture the // lexical name. const universalThis = globalThis; diff --git a/packages/ses/src/compartment-shim.js b/packages/ses/src/compartment-shim.js index ee3ddface4..5708c6f3dc 100644 --- a/packages/ses/src/compartment-shim.js +++ b/packages/ses/src/compartment-shim.js @@ -16,4 +16,7 @@ globalThis.Compartment = makeCompartmentConstructor( // See https://github.com/endojs/endo/pull/2624#discussion_r1840979770 getGlobalIntrinsics(globalThis, muteReporter), markVirtualizedNativeFunction, + { + enforceNew: true, + }, ); diff --git a/packages/ses/src/compartment.js b/packages/ses/src/compartment.js index c423a48f56..6371eeac92 100644 --- a/packages/ses/src/compartment.js +++ b/packages/ses/src/compartment.js @@ -1,3 +1,40 @@ +/** + * @module Provides the mechanism to create a Compartment constructor that + * can provide either shim-specific or native XS features depending on + * the __native__ constructor option. + * This is necessary because a native Compartment can handle native ModuleSource + * but cannot handle shim-specific pre-compiled ModuleSources like the JSON + * representation of a module that Compartment Mapper can put in bundles. + * Pre-compiling ModuleSource during bundling helps avoid paying the cost + * of importing Babel and transforming ESM syntax to a form that can be + * confined by the shim, which is prohibitively expensive for a web runtime + * and for XS _without this adapter_. + * + * Since any invocation of the Compartment constructor may occur standing + * on a native-flavor or shim-flavor compartment, we create parallel compartment + * constructor trees for compartments created with the Compartment constructor + * of a specific compartment. + * + * A compartment's importHook, importNowHook, moduleMapHook, and the modules + * map itself may provide module descriptors that address another compartment, + * using a compartment instance as a token indicating the compartment the + * module should be loaded or initialized in. + * Consequently, the compartment instance must be a suitable token for the + * underlying native-flavor or shim-flavor compartment. + * We are not in a position to fidddle with the native compartments behavior, + * so adapted compartments use the identity of the native compartment. + * We replace all of the methods of the native compartment prototype with + * thunks that choose behavior based on whether the compartment was + * constructed with the __native__ option. + * The SES shim associates a compartment with its private fields using a weak + * map exported by ../src/compartment.js and held closely by ses by the + * enforcement of explicit exports in package.json, since Node.js 12.11.0. + * + * Evaluating ./compartment.js does not have global side-effects. + * We defer modification of the global environment until the evaluation + * of ./compartment-shim.js. + */ + // @ts-check /* eslint-disable no-underscore-dangle */ /// @@ -171,7 +208,9 @@ defineProperties(InertCompartment, { * @param {MakeCompartmentConstructor} targetMakeCompartmentConstructor * @param {Record} intrinsics * @param {(object: object) => void} markVirtualizedNativeFunction - * @param {Compartment} [parentCompartment] + * @param {object} [options] + * @param {Compartment} [options.parentCompartment] + * @param {boolean} [options.enforceNew] * @returns {Compartment['constructor']} */ @@ -184,7 +223,7 @@ defineProperties(InertCompartment, { // positional arguments, this function detects the temporary sigil __options__ // on the first argument and coerces compartments arguments into a single // compartments object. -const compartmentOptions = (...args) => { +export const compartmentOptions = (...args) => { if (args.length === 0) { return {}; } @@ -229,10 +268,10 @@ export const makeCompartmentConstructor = ( targetMakeCompartmentConstructor, intrinsics, markVirtualizedNativeFunction, - parentCompartment = undefined, + { parentCompartment = undefined, enforceNew = false } = {}, ) => { function Compartment(...args) { - if (new.target === undefined) { + if (enforceNew && new.target === undefined) { throw TypeError( "Class constructor Compartment cannot be invoked without 'new'", ); diff --git a/packages/ses/src/global-object.js b/packages/ses/src/global-object.js index 007399fa5c..1b2c3c038d 100644 --- a/packages/ses/src/global-object.js +++ b/packages/ses/src/global-object.js @@ -117,7 +117,10 @@ export const setGlobalObjectMutableProperties = ( makeCompartmentConstructor, intrinsics, markVirtualizedNativeFunction, - parentCompartment, + { + parentCompartment, + enforceNew: true, + }, ), ); diff --git a/packages/ses/test/_meaning.js b/packages/ses/test/_meaning.js new file mode 100644 index 0000000000..7a4e8a723a --- /dev/null +++ b/packages/ses/test/_meaning.js @@ -0,0 +1 @@ +export default 42; diff --git a/packages/ses/test/_xs.js b/packages/ses/test/_xs.js new file mode 100644 index 0000000000..64f0b14e3c --- /dev/null +++ b/packages/ses/test/_xs.js @@ -0,0 +1,226 @@ +// This is a test fixture for minimal spot checks of the XS-specific variant of +// SES. +// The script ../scripts/generate-test-xs.js generates the _meaning.pre-mjs.json +// module by precompiling _meaning.js, then bundles this module with the "xs" +// package export/import condition so that it entrains ../src-xs/shim.js instead +// of the ordinary SES shim. +// This generates ../tmp/test-xs.js, which can be run with xst directly for +// validation of the XS environment under SES-for-XS. + +/* global print */ + +// Eslint does not know about package reflexive imports (importing your own +// package), which in this case is necessary to go through the conditional +// export in package.json. +// eslint-disable-next-line import/no-extraneous-dependencies +import 'ses'; + +// The dependency below is generated by ../scripts/generate-test-xs.js +// eslint-disable-next-line import/no-unresolved +import precompiledModuleSource from '../tmp/_meaning.pre-mjs.json'; + +lockdown(); + +// spot checks +assert(Object.isFrozen(Object)); + +print('# shim compartment can import a shim precompiled module source'); +{ + const shimCompartment = new Compartment({ + __options__: true, + modules: { + '.': { + source: precompiledModuleSource, + }, + }, + }); + assert.equal( + shimCompartment.importNow('.').default, + 42, + 'can import precompiled module source', + ); +} + +print('# native compartment can import a native ModuleSource'); +{ + const nativeCompartment = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + source: new ModuleSource(` + export default 42; + `), + }, + }, + }); + + assert( + nativeCompartment.importNow('.').default === 42, + 'can import native module source', + ); +} + +print('# shim compartment cannot import a native ModuleSource'); +// fail to import a native module source in a shim compartment +{ + let threw = null; + try { + new Compartment({ + __options__: true, + modules: { + '.': { + source: new ModuleSource(''), + }, + }, + }).importNow('.'); + } catch (error) { + threw = error; + } + assert( + threw, + 'attempting to import a native module source on a shim compartment should fail', + ); +} + +print('# native compartment cannot import a shim precompiled module source'); +{ + let threw = null; + try { + new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + source: precompiledModuleSource, + }, + }, + }).importNow('.'); + } catch (error) { + threw = error; + } + assert( + threw, + 'attempting to import a precompiled module source in a native compartment should fail', + ); +} + +print('# shim compartment can link to another shim compartment'); +{ + const shimCompartment1 = new Compartment({ + __options__: true, + modules: { + '.': { + source: precompiledModuleSource, + }, + }, + }); + const shimCompartment2 = new Compartment({ + __options__: true, + modules: { + '.': { + compartment: shimCompartment1, + namespace: '.', + }, + }, + }); + assert.equal( + shimCompartment2.importNow('.').default, + 42, + 'can link shim compartments', + ); +} + +print('# native compartment can link to another native compartment'); +{ + const nativeCompartment1 = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + source: new ModuleSource(` + export default 42; + `), + }, + }, + }); + const nativeCompartment2 = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + compartment: nativeCompartment1, + namespace: '.', + }, + }, + }); + assert.equal( + nativeCompartment2.importNow('.').default, + 42, + 'can link native compartments', + ); +} + +print('# shim compartment cannot link a native compartment'); +{ + const nativeCompartment = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + source: new ModuleSource(` + export default 42; + `), + }, + }, + }); + const shimCompartment = new Compartment({ + __options__: true, + modules: { + '.': { + compartment: nativeCompartment, + namespace: '.', + }, + }, + }); + let threw = null; + try { + shimCompartment.importNow('.'); + } catch (error) { + threw = error; + } + assert(threw, 'cannot link native from shim compartment'); +} + +print('# native compartment cannot link shim compartment'); +{ + const shimCompartment = new Compartment({ + __options__: true, + modules: { + '.': { + source: precompiledModuleSource, + }, + }, + }); + const nativeCompartment = new Compartment({ + __options__: true, + __native__: true, + modules: { + '.': { + compartment: shimCompartment, + namespace: '.', + }, + }, + }); + let threw = null; + try { + nativeCompartment.importNow('.'); + } catch (error) { + threw = error; + } + assert(threw, 'cannot link shim from native compartment'); +} + +print('ok'); + +// To be continued in hardened262...