const vm = require('vm')
const fs = require('fs')
const path = require('path')
const jsYaml = require('js-yaml')
const isatty = require('tty').isatty(process.stdout.fd)
const { installForTests } = require('./esbuild')
const testDir = path.join(__dirname, '..', 'demo', 'test262', 'test')
const harnessDir = path.join(__dirname, '..', 'demo', 'test262', 'harness')
const progressBarLength = 64
const eraseProgressBar = () => {
previousProgressBar = null
let text = `\r${' '.repeat(progressBarLength)}\r`
if (printNewlineWhenErasing) {
printNewlineWhenErasing = false
text += '\n'
}
return text
}
let previousProgressBar = null
let printNewlineWhenErasing = false
const resetColor = isatty ? `\x1b[0m` : ''
const boldColor = isatty ? `\x1b[1m` : ''
const dimColor = isatty ? `\x1b[37m` : ''
const underlineColor = isatty ? `\x1b[4m` : ''
const redColor = isatty ? `\x1b[31m` : ''
const greenColor = isatty ? `\x1b[32m` : ''
const blueColor = isatty ? `\x1b[34m` : ''
const cyanColor = isatty ? `\x1b[36m` : ''
const magentaColor = isatty ? `\x1b[35m` : ''
const yellowColor = isatty ? `\x1b[33m` : ''
const redBgRedColor = isatty ? `\x1b[41;31m` : ''
const redBgWhiteColor = isatty ? `\x1b[41;97m` : ''
const greenBgGreenColor = isatty ? `\x1b[42;32m` : ''
const greenBgWhiteColor = isatty ? `\x1b[42;97m` : ''
const blueBgBlueColor = isatty ? `\x1b[44;34m` : ''
const blueBgWhiteColor = isatty ? `\x1b[44;97m` : ''
const cyanBgCyanColor = isatty ? `\x1b[46;36m` : ''
const cyanBgBlackColor = isatty ? `\x1b[46;30m` : ''
const magentaBgMagentaColor = isatty ? `\x1b[45;35m` : ''
const magentaBgBlackColor = isatty ? `\x1b[45;30m` : ''
const yellowBgYellowColor = isatty ? `\x1b[43;33m` : ''
const yellowBgBlackColor = isatty ? `\x1b[43;30m` : ''
const whiteBgWhiteColor = isatty ? `\x1b[107;97m` : ''
const whiteBgBlackColor = isatty ? `\x1b[107;30m` : ''
const skipTheseFeatures = new Set([
'decorators',
'regexp-v-flag',
'regexp-match-indices',
'regexp-named-groups',
'regexp-unicode-property-escapes',
])
const skipTheseTests = new Set([
'language/expressions/import.meta/syntax/goal-script.js',
'language/global-code/export.js',
'language/global-code/import.js',
'language/statements/return/S12.9_A1_T1.js',
'language/statements/return/S12.9_A1_T10.js',
'language/statements/return/S12.9_A1_T2.js',
'language/statements/return/S12.9_A1_T3.js',
'language/statements/return/S12.9_A1_T4.js',
'language/statements/return/S12.9_A1_T5.js',
'language/statements/return/S12.9_A1_T6.js',
'language/statements/return/S12.9_A1_T7.js',
'language/statements/return/S12.9_A1_T8.js',
'language/statements/return/S12.9_A1_T9.js',
'language/global-code/return.js',
'language/expressions/assignmenttargettype/simple-basic-identifierreference-await.js',
'language/expressions/await/await-BindingIdentifier-in-global.js',
'language/expressions/await/await-in-global.js',
'language/expressions/await/await-in-nested-function.js',
'language/expressions/await/await-in-nested-generator.js',
'language/expressions/class/class-name-ident-await-escaped.js',
'language/expressions/class/class-name-ident-await.js',
'language/expressions/class/static-init-await-reference.js',
'language/expressions/dynamic-import/2nd-param-await-ident.js',
'language/expressions/dynamic-import/assignment-expression/await-identifier.js',
'language/expressions/function/static-init-await-reference.js',
'language/expressions/generators/static-init-await-reference.js',
'language/expressions/in/private-field-rhs-await-absent.js',
'language/expressions/object/identifier-shorthand-await-strict-mode.js',
'language/expressions/object/method-definition/static-init-await-reference-accessor.js',
'language/expressions/object/method-definition/static-init-await-reference-generator.js',
'language/expressions/object/method-definition/static-init-await-reference-normal.js',
'language/module-code/top-level-await/new-await-script-code.js',
'language/reserved-words/await-script.js',
'language/statements/class/class-name-ident-await-escaped.js',
'language/statements/class/class-name-ident-await.js',
'language/statements/labeled/value-await-non-module-escaped.js',
'language/statements/labeled/value-await-non-module.js',
'language/literals/regexp/early-err-pattern.js',
'language/literals/regexp/invalid-braced-quantifier-exact.js',
'language/literals/regexp/invalid-braced-quantifier-lower.js',
'language/literals/regexp/invalid-braced-quantifier-range.js',
'language/literals/regexp/invalid-optional-lookbehind.js',
'language/literals/regexp/invalid-optional-negative-lookbehind.js',
'language/literals/regexp/invalid-range-lookbehind.js',
'language/literals/regexp/invalid-range-negative-lookbehind.js',
'language/literals/regexp/u-invalid-class-escape.js',
'language/literals/regexp/u-invalid-extended-pattern-char.js',
'language/literals/regexp/u-invalid-identity-escape.js',
'language/literals/regexp/u-invalid-legacy-octal-escape.js',
'language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-a.js',
'language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-ab.js',
'language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-b.js',
'language/literals/regexp/u-invalid-non-empty-class-ranges.js',
'language/literals/regexp/u-invalid-oob-decimal-escape.js',
'language/literals/regexp/u-invalid-optional-lookahead.js',
'language/literals/regexp/u-invalid-optional-lookbehind.js',
'language/literals/regexp/u-invalid-optional-negative-lookahead.js',
'language/literals/regexp/u-invalid-optional-negative-lookbehind.js',
'language/literals/regexp/u-invalid-range-lookahead.js',
'language/literals/regexp/u-invalid-range-lookbehind.js',
'language/literals/regexp/u-invalid-range-negative-lookahead.js',
'language/literals/regexp/u-invalid-range-negative-lookbehind.js',
'language/literals/regexp/u-unicode-esc-bounds.js',
'language/literals/regexp/u-unicode-esc-non-hex.js',
'language/literals/regexp/unicode-escape-nls-err.js',
])
const skipEvaluatingTheseIncludes = new Set([
'nativeFunctionMatcher.js',
])
const skipEvaluatingTheseFeatures = new Set([
'hashbang',
'legacy-regexp',
'regexp-duplicate-named-groups',
'symbols-as-weakmap-keys',
'tail-call-optimization',
'ArrayBuffer',
'change-array-by-copy',
'DataView',
'resizable-arraybuffer',
'ShadowRealm',
'SharedArrayBuffer',
'String.prototype.toWellFormed',
'Symbol.match',
'Symbol.replace',
'Symbol.unscopables',
'Temporal',
'TypedArray',
])
const skipEvaluatingTheseTests = new Set([
'language/expressions/dynamic-import/2nd-param-assert-enumeration.js',
])
function findFiles() {
function visit(dir) {
for (const entry of fs.readdirSync(dir)) {
const fullEntry = path.join(dir, entry)
const stats = fs.statSync(fullEntry)
if (stats.isDirectory()) {
visit(fullEntry)
} else if (stats.isFile() && entry.endsWith('.js') && !entry.includes('_FIXTURE')) {
files.push(fullEntry)
}
}
}
const files = []
for (const entry of fs.readdirSync(testDir)) {
if (entry === 'staging' || entry === 'intl402' || entry === 'built-ins') continue
visit(path.join(testDir, entry))
}
return files.reverse()
}
async function checkTransformAPI({ esbuild, file, content, yaml }) {
if (yaml.flags) {
if (yaml.flags.includes('onlyStrict')) content = '"use strict";' + content
if (yaml.flags.includes('module')) content += '\nexport {}'
}
const shouldParse = !yaml.negative || yaml.negative.phase !== 'parse'
let result
try {
result = await esbuild.transform(content, { sourcefile: file })
} catch (error) {
if (shouldParse) {
error.kind = 'Transform'
throw error
}
return
}
if (!shouldParse) {
const error = new Error('Unexpected successful transform')
error.kind = 'Transform'
throw error
}
let result2
try {
result2 = await esbuild.transform(result.code, { sourcefile: file })
} catch (error) {
error.kind = 'Reparse'
throw error
}
if (result2.code !== result.code) {
const lines = result.code.split('\n')
const lines2 = result2.code.split('\n')
let i = 0
while (i < lines.length && i < lines2.length && lines[i] === lines2[i]) i++
const error = { toString: () => `${redColor}-${lines[i]}\n${greenColor}+${lines2[i]}${resetColor}` }
error.kind = 'Reprint'
throw error
}
let result4
try {
result4 = await esbuild.transform(result2.code, { sourcefile: file, minify: true })
} catch (error) {
error.kind = 'Minify'
throw error
}
let result5
try {
result5 = await esbuild.transform(result4.code, { sourcefile: file, minify: true })
} catch (error) {
error.kind = 'Minify'
throw error
}
}
async function checkBuildAPI({ esbuild, file, content, yaml }) {
const plugins = []
if (yaml.flags) {
const isOnlyStrict = yaml.flags.includes('onlyStrict')
const isModule = yaml.flags.includes('module')
if (isOnlyStrict || isModule) {
plugins.push({
name: 'modify',
setup(build) {
build.onLoad({ filter: /./ }, args => {
if (args.path === file) {
let loaded = content
if (isOnlyStrict) loaded = '"use strict";' + loaded
if (isModule) loaded += '\nexport {}'
return { contents: loaded }
}
})
},
})
}
}
const isModule = yaml.flags && yaml.flags.includes('module')
const isDynamicImport = yaml.flags && yaml.flags.includes('dynamic-import')
const isAsync = yaml.flags && yaml.flags.includes('async')
const shouldParse = !yaml.negative || yaml.negative.phase === 'runtime'
let result
try {
const options = {
entryPoints: [file],
write: false,
keepNames: true,
logLevel: 'silent',
plugins,
target: 'node' + process.version.slice(1),
logOverride: { 'direct-eval': 'warning' },
}
if (isModule || isDynamicImport || isAsync) {
options.bundle = true
options.format = isModule ? 'esm' : 'iife'
options.external = [
'',
]
}
result = await esbuild.build(options)
} catch (error) {
if (shouldParse) {
error.kind = 'Build'
throw error
}
return
}
if (!shouldParse) {
const error = new Error('Unexpected successful build')
error.kind = 'Build'
throw error
}
const hasDirectEval = result.warnings.some(msg => msg.id === 'direct-eval')
if (
hasDirectEval ||
skipEvaluatingTheseTests.has(path.relative(testDir, file)) ||
(yaml.includes && yaml.includes.some(include => skipEvaluatingTheseIncludes.has(include))) ||
(yaml.features && yaml.features.some(feature => skipEvaluatingTheseFeatures.has(feature)))
) {
return
}
const importDir = path.dirname(file)
const shouldEvaluate = !yaml.negative
try {
await runCodeInHarness(yaml, content, importDir)
} catch (error) {
if (shouldEvaluate) console.log(eraseProgressBar() + dimColor + `IGNORING ${path.relative(testDir, file)}: ${error}` + resetColor)
return
}
if (!shouldEvaluate) {
return
}
const code = result.outputFiles[0].text
try {
await runCodeInHarness(yaml, code, importDir)
} catch (error) {
if (shouldEvaluate) {
if (typeof error === 'string') error = new Error(error)
error.kind = 'Evaluate'
throw error
}
return
}
if (!shouldEvaluate) {
const error = new Error('Unexpected successful evaluation')
error.kind = 'Evaluate'
throw error
}
for (let version = 2015; version <= 2022; version++) {
let result
try {
result = await esbuild.transform(code, { sourcefile: file, target: `es${version}` })
} catch (error) {
continue
}
try {
await runCodeInHarness(yaml, code, importDir)
} catch (error) {
if (typeof error === 'string') error = new Error(error)
error.kind = 'Lower'
throw error
}
break
}
}
async function main() {
const startTime = Date.now()
{
const temp = new vm.SourceTextModule('')
await temp.link(() => { throw new Error })
await temp.evaluate()
}
console.log(`\n${dimColor}Finding tests...${resetColor}`)
const files = findFiles()
console.log(`Found ${files.length} test files`)
console.log(`\n${dimColor}Installing esbuild...${resetColor}`)
const esbuild = installForTests()
console.log(`\n${dimColor}Running tests...${resetColor}\n`)
const errorCounts = {}
let skippedCount = 0
await forEachInParallel(files, 32, async (file) => {
if (skipTheseTests.has(path.relative(testDir, file))) {
skippedCount++
return
}
try {
const content = fs.readFileSync(file, 'utf8')
const start = content.indexOf('/*---')
const end = content.indexOf('---*/')
if (start < 0 || end < 0) throw new Error(`Missing YAML metadata`)
const yaml = jsYaml.safeLoad(content.slice(start + 5, end))
if (yaml.features && yaml.features.some(feature => skipTheseFeatures.has(feature))) {
skippedCount++
return
}
await checkTransformAPI({ esbuild, file, content, yaml })
await checkBuildAPI({ esbuild, file, content, yaml })
}
catch (error) {
errorCounts[error.kind] = (errorCounts[error.kind] || 0) + 1
printError(file, error)
}
})
const table = []
table.push(['Total tests', `${files.length}`])
table.push(['Tests ran', `${files.length - skippedCount}`])
table.push(['Tests skipped', `${skippedCount}`])
for (const kind of Object.keys(errorCounts).sort()) {
table.push([kind + ' errors', `${errorCounts[kind]}`])
}
const seconds = (Date.now() - startTime) / 1000
const minutes = Math.floor(seconds / 60)
table.push(['Time taken', `${minutes ? `${minutes} min ${+(seconds - minutes * 60).toFixed(1)} sec` : `${+seconds.toFixed(1)} sec`}`])
const maxLength = Math.max(...table.map(x => x[0].length))
printNewlineWhenErasing = true
process.stdout.write(eraseProgressBar())
for (const [key, value] of table) {
console.log(`${boldColor}${(key + ':').padEnd(maxLength + 1)}${resetColor} ${value}`)
}
}
function forEachInParallel(items, batchSize, callback) {
return new Promise((resolve, reject) => {
let inFlight = 0
let i = 0
function next() {
if (i === items.length && inFlight === 0) {
process.stdout.write(eraseProgressBar())
return resolve()
}
const completed = Math.floor(progressBarLength * i / items.length)
if (previousProgressBar !== completed) {
previousProgressBar = completed
const progressHead = '\u2501'.repeat(Math.max(0, completed - 1))
const progressBoundary = completed ? '\u252B' : ''
const progressTail = '\u2500'.repeat(progressBarLength - completed)
process.stdout.write(`\r` + greenColor + progressHead + progressBoundary + dimColor + progressTail + resetColor)
}
while (i < items.length && inFlight < batchSize) {
inFlight++
callback(items[i++]).then(() => {
inFlight--
next()
}, reject)
}
}
next()
})
}
const harnessFiles = new Map
let defaultHarness = ''
for (const entry of fs.readdirSync(harnessDir)) {
if (entry.startsWith('.') || !entry.endsWith('.js')) {
continue
}
const file = path.join(harnessDir, entry)
const content = fs.readFileSync(file, 'utf8')
if (entry === 'assert.js' || entry === 'sta.js') {
defaultHarness += content
continue
}
harnessFiles.set(entry, content)
}
function createHarnessForTest(yaml) {
let harness = defaultHarness
if (yaml.includes) {
for (const include of yaml.includes) {
const content = harnessFiles.get(include)
if (!content) throw new Error(`Included file is missing: ${include}`)
harness += content
}
}
return harness
}
async function runCodeInHarness(yaml, code, importDir) {
const context = {}
const isAsync = yaml.flags && yaml.flags.includes('async')
const isModule = yaml.flags && yaml.flags.includes('module')
const isRaw = yaml.flags && yaml.flags.includes('raw')
const unique = () => '//' + Math.random()
const runCode = async () => {
const moduleCache = new Map
const dynamicImportCache = new Map
const findModule = (modulePath) => {
let module = moduleCache.get(modulePath)
if (!module) {
const code = fs.readFileSync(modulePath, 'utf8')
if (modulePath.endsWith('json')) {
const evaluate = function () {
this.setExport('default', vm.runInContext('JSON.parse', context)(code))
}
module = new vm.SyntheticModule(['default'], evaluate, { context })
} else {
module = new vm.SourceTextModule(code + unique(), { context, importModuleDynamically })
}
moduleCache.set(modulePath, module)
}
return module
}
const linker = (specifier, referencingModule) => {
return findModule(path.join(importDir, specifier))
}
const importModuleDynamically = (specifier, script) => {
const where = path.join(importDir, specifier)
let promise = dynamicImportCache.get(where)
if (!promise) {
const module = findModule(where, context)
if (module.status === 'unlinked') {
promise = module.link(linker)
.then(() => module.evaluate())
.then(() => module)
} else {
promise = Promise.resolve(module)
}
dynamicImportCache.set(where, promise)
}
return promise
}
vm.createContext(context)
if (!isRaw) vm.runInContext(createHarnessForTest(yaml), context)
if (isModule) {
const module = new vm.SourceTextModule(code + unique(), { context, importModuleDynamically })
await module.link(linker)
await module.evaluate()
} else {
const script = new vm.Script(code, { importModuleDynamically })
script.runInContext(context)
}
}
if (isAsync) {
await new Promise((resolve, reject) => {
context.$DONE = err => err ? reject(err) : resolve()
runCode(code, context).catch(reject)
})
} else {
await runCode(code, context)
}
}
function printError(file, error) {
let detail
if (error.errors) {
const { text, location } = error.errors[0]
if (location) {
const { file, line, column, lineText, length } = location
detail = ' ' + dimColor + path.basename(file) + ':' + line + ':' + column + ': ' + resetColor + text + '\n' +
' ' + dimColor + lineText.slice(0, column) + greenColor + lineText.slice(column, column + length) + dimColor + lineText.slice(column + length) + resetColor + '\n' +
' ' + greenColor + ' '.repeat(column) + (length > 1 ? '~'.repeat(length) : '^') + resetColor
} else {
detail = dimColor + ('\n' + text).split('\n').join('\n ').slice(1) + resetColor
}
} else {
detail = dimColor + ('\n' + error).split('\n').join('\n ').slice(1) + resetColor
}
const prettyPath = path.relative(testDir, file)
printNewlineWhenErasing = true
console.log(eraseProgressBar() + tagMap[error.kind] + ' ' + prettyPath + '\n' + detail)
printNewlineWhenErasing = true
}
const tagMap = {
Transform: redBgRedColor + `[` + redBgWhiteColor + `TRANSFORM ERROR` + redBgRedColor + `]` + resetColor,
Build: magentaBgMagentaColor + `[` + magentaBgBlackColor + `BUILD ERROR` + magentaBgMagentaColor + `]` + resetColor,
Reparse: yellowBgYellowColor + `[` + yellowBgBlackColor + `REPARSE ERROR` + yellowBgYellowColor + `]` + resetColor,
Reprint: cyanBgCyanColor + `[` + cyanBgBlackColor + `REPRINT ERROR` + cyanBgCyanColor + `]` + resetColor,
Minify: blueBgBlueColor + `[` + blueBgWhiteColor + `MINIFY ERROR` + blueBgBlueColor + `]` + resetColor,
Evaluate: greenBgGreenColor + `[` + greenBgWhiteColor + `EVALUATE ERROR` + greenBgGreenColor + `]` + resetColor,
Lower: whiteBgWhiteColor + `[` + whiteBgBlackColor + `LOWER ERROR` + whiteBgWhiteColor + `]` + resetColor,
}
process.on('unhandledRejection', () => {
})
main().catch(e => setTimeout(() => {
throw e
}))