Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updates vfs to be more future safe when running on node + 2021 dts files for playground #2802

Merged
merged 11 commits into from
Jul 25, 2023
Merged

Updates vfs to be more future safe when running on node + 2021 dts files for playground #2802

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
a2e8661
Updates vfs to be more future safe when running on node, and to also …
orta Apr 25, 2023
1d0bc44
Take tsLibDirectory into account
orta Apr 25, 2023
b5187a7
Update lockfile
orta Apr 26, 2023
d1dd3ca
Include all dts files ever shipped
orta Apr 28, 2023
438d263
Handle the CDN potentially causing fetch throws
orta May 2, 2023
0488958
Replace with new list
jakebailey Jul 25, 2023
fefe86e
Merge branch 'v2' into pr-2802
jakebailey Jul 25, 2023
c2feaab
Undo package.json changes, will push in a different PR
jakebailey Jul 25, 2023
abd38b7
Maybe order matters
jakebailey Jul 25, 2023
82a915f
Maybe order matters again
jakebailey Jul 25, 2023
ec60dc5
Merge branch 'v2' into pr-2802
jakebailey Jul 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/sandbox/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
},
"dependencies": {
"@typescript/ata": "0.9.3",
"@typescript/vfs": "1.4.0"
"@typescript/vfs": "1.5.0"
},
"devDependencies": {
"@rollup/plugin-commonjs": "^13.0.1",
Expand Down
6 changes: 3 additions & 3 deletions packages/ts-twoslasher/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@typescript/twoslash",
"version": "3.2.1",
"version": "3.2.2",
"license": "MIT",
"author": "TypeScript team",
"homepage": "https://github.com/microsoft/TypeScript-Website",
Expand Down Expand Up @@ -53,8 +53,8 @@
"typescript": false
},
"dependencies": {
"@typescript/vfs": "1.4.0",
"@typescript/vfs": "1.5.0",
"debug": "^4.1.1",
"lz-string": "^1.4.4"
}
}
}
5 changes: 5 additions & 0 deletions packages/typescript-vfs/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
### 1.5

- Makes `createDefaultMapFromNodeModules` pull all the `.d.ts` files from the node_modules folder, not just the `.d.ts` files which were known ahead of time.
- Updates the known .d.s files to include ones from TypeScript 5.1 beta.

### 1.3

- Adds a JS file into the npm tarball for using with a vanilla script tag, which sets `global.tsvfs` with exported function.
Expand Down
2 changes: 1 addition & 1 deletion packages/typescript-vfs/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@typescript/vfs",
"version": "1.4.0",
"version": "1.5.0",
"license": "MIT",
"author": "TypeScript team",
"homepage": "https://github.com/microsoft/TypeScript-Website",
Expand Down
178 changes: 109 additions & 69 deletions packages/typescript-vfs/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ type TS = typeof import("typescript")
let hasLocalStorage = false
try {
hasLocalStorage = typeof localStorage !== `undefined`
} catch (error) { }
} catch (error) {}

const hasProcess = typeof process !== `undefined`
const shouldDebug = (hasLocalStorage && localStorage.getItem("DEBUG")) || (hasProcess && process.env.DEBUG)
Expand Down Expand Up @@ -91,9 +91,13 @@ export function createVirtualTypeScriptEnvironment(
}
}

// TODO: This could be replaced by grabbing: https://github.com/microsoft/TypeScript/blob/main/src/lib/libs.json
// and then using that to generate the list of files from the server, but it is not included in the npm package
jakebailey marked this conversation as resolved.
Show resolved Hide resolved

/**
* Grab the list of lib files for a particular target, will return a bit more than necessary (by including
* the dom) but that's OK
* the dom) but that's OK, we're really working with the constraint that you can't get a list of files
* when running in a browser.
*
* @param target The compiler settings target baseline
* @param ts A copy of the TypeScript module
Expand All @@ -102,68 +106,91 @@ export const knownLibFilesForCompilerOptions = (compilerOptions: CompilerOptions
const target = compilerOptions.target || ts.ScriptTarget.ES5
const lib = compilerOptions.lib || []

const files = [
"lib.d.ts",
"lib.dom.d.ts",
"lib.dom.iterable.d.ts",
"lib.webworker.d.ts",
"lib.webworker.importscripts.d.ts",
"lib.scripthost.d.ts",
"lib.es5.d.ts",
"lib.es6.d.ts",
"lib.es2015.collection.d.ts",
"lib.es2015.core.d.ts",
"lib.es2015.d.ts",
"lib.es2015.generator.d.ts",
"lib.es2015.iterable.d.ts",
"lib.es2015.promise.d.ts",
"lib.es2015.proxy.d.ts",
"lib.es2015.reflect.d.ts",
"lib.es2015.symbol.d.ts",
"lib.es2015.symbol.wellknown.d.ts",
"lib.es2016.array.include.d.ts",
"lib.es2016.d.ts",
"lib.es2016.full.d.ts",
"lib.es2017.d.ts",
"lib.es2017.full.d.ts",
"lib.es2017.intl.d.ts",
"lib.es2017.object.d.ts",
"lib.es2017.sharedmemory.d.ts",
"lib.es2017.string.d.ts",
"lib.es2017.typedarrays.d.ts",
"lib.es2018.asyncgenerator.d.ts",
"lib.es2018.asynciterable.d.ts",
"lib.es2018.d.ts",
"lib.es2018.full.d.ts",
"lib.es2018.intl.d.ts",
"lib.es2018.promise.d.ts",
"lib.es2018.regexp.d.ts",
"lib.es2019.array.d.ts",
"lib.es2019.d.ts",
"lib.es2019.full.d.ts",
"lib.es2019.object.d.ts",
"lib.es2019.string.d.ts",
"lib.es2019.symbol.d.ts",
"lib.es2020.d.ts",
"lib.es2020.full.d.ts",
"lib.es2020.string.d.ts",
"lib.es2020.symbol.wellknown.d.ts",
"lib.es2020.bigint.d.ts",
"lib.es2020.promise.d.ts",
"lib.es2020.sharedmemory.d.ts",
"lib.es2020.intl.d.ts",
"lib.es2021.d.ts",
"lib.es2021.full.d.ts",
"lib.es2021.promise.d.ts",
"lib.es2021.string.d.ts",
"lib.es2021.weakref.d.ts",
"lib.esnext.d.ts",
"lib.esnext.full.d.ts",
"lib.esnext.intl.d.ts",
"lib.esnext.promise.d.ts",
"lib.esnext.string.d.ts",
"lib.esnext.weakref.d.ts",
]
// Note that this will include files which can't be found for particular versions of TS
const files =
"getAllLibFileNames" in ts
? // @ts-ignore - see https://github.com/microsoft/TypeScript/pull/54011
(ts.getAllLibFileNames() as string[])
: [
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this list get filtered in any way to the actual set based on what's available? I'm not sure I totally know how this list is used.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, there's a rough heuristic which cuts down the list based on target + lib - just below:

  const targetToCut = ts.ScriptTarget[target]
  const matches = files.filter(f => f.startsWith(`lib.${targetToCut.toLowerCase()}`))
  const targetCutIndex = files.indexOf(matches.pop()!)

  const getMax = (array: number[]) =>
    array && array.length ? array.reduce((max, current) => (current > max ? current : max)) : undefined

  // Find the index for everything in
  const indexesForCutting = lib.map(lib => {
    const matches = files.filter(f => f.startsWith(`lib.${lib.toLowerCase()}`))
    if (matches.length === 0) return 0

    const cutIndex = files.indexOf(matches.pop()!)
    return cutIndex
  })

  const libCutIndex = getMax(indexesForCutting) || 0

  const finalCutIndex = Math.max(targetCutIndex, libCutIndex)
  return files.slice(0, finalCutIndex + 1)
}

So the order is important WRT downloading which have some tests covering the cases, but the rough gist is that the CDN version will download all of the files (and cache them locally if possible) and put them into the VFS which TypeScript will then use synchronously later

I just added some checks for handle CDNs returning NOOPs for dts files which dont exist for that ts version

"lib.d.ts",
"lib.decorators.d.ts",
"lib.decorators.legacy.d.ts",
"lib.dom.d.ts",
"lib.dom.iterable.d.ts",
"lib.es2015.collection.d.ts",
"lib.es2015.core.d.ts",
"lib.es2015.d.ts",
"lib.es2015.generator.d.ts",
"lib.es2015.iterable.d.ts",
"lib.es2015.promise.d.ts",
"lib.es2015.proxy.d.ts",
"lib.es2015.reflect.d.ts",
"lib.es2015.symbol.d.ts",
"lib.es2015.symbol.wellknown.d.ts",
"lib.es2016.array.include.d.ts",
"lib.es2016.d.ts",
"lib.es2016.full.d.ts",
"lib.es2017.d.ts",
"lib.es2017.full.d.ts",
"lib.es2017.intl.d.ts",
"lib.es2017.object.d.ts",
"lib.es2017.sharedmemory.d.ts",
"lib.es2017.string.d.ts",
"lib.es2017.typedarrays.d.ts",
"lib.es2018.asyncgenerator.d.ts",
"lib.es2018.asynciterable.d.ts",
"lib.es2018.d.ts",
"lib.es2018.full.d.ts",
"lib.es2018.intl.d.ts",
"lib.es2018.promise.d.ts",
"lib.es2018.regexp.d.ts",
"lib.es2019.array.d.ts",
"lib.es2019.d.ts",
"lib.es2019.full.d.ts",
"lib.es2019.intl.d.ts",
"lib.es2019.object.d.ts",
"lib.es2019.string.d.ts",
"lib.es2019.symbol.d.ts",
"lib.es2020.bigint.d.ts",
"lib.es2020.d.ts",
"lib.es2020.date.d.ts",
"lib.es2020.full.d.ts",
"lib.es2020.intl.d.ts",
"lib.es2020.number.d.ts",
"lib.es2020.promise.d.ts",
"lib.es2020.sharedmemory.d.ts",
"lib.es2020.string.d.ts",
"lib.es2020.symbol.wellknown.d.ts",
"lib.es2021.d.ts",
"lib.es2021.full.d.ts",
"lib.es2021.intl.d.ts",
"lib.es2021.promise.d.ts",
"lib.es2021.string.d.ts",
"lib.es2021.weakref.d.ts",
"lib.es2022.array.d.ts",
"lib.es2022.d.ts",
"lib.es2022.error.d.ts",
"lib.es2022.full.d.ts",
"lib.es2022.intl.d.ts",
"lib.es2022.object.d.ts",
"lib.es2022.regexp.d.ts",
"lib.es2022.sharedmemory.d.ts",
"lib.es2022.string.d.ts",
"lib.es2023.array.d.ts",
"lib.es2023.d.ts",
"lib.es2023.full.d.ts",
"lib.es5.d.ts",
"lib.es6.d.ts",
"lib.esnext.d.ts",
"lib.esnext.full.d.ts",
"lib.esnext.intl.d.ts",
"lib.esnext.promise.d.ts",
"lib.esnext.string.d.ts",
"lib.esnext.weakref.d.ts",
"lib.scripthost.d.ts",
"lib.webworker.d.ts",
"lib.webworker.importscripts.d.ts",
]

const targetToCut = ts.ScriptTarget[target]
const matches = files.filter(f => f.startsWith(`lib.${targetToCut.toLowerCase()}`))
Expand All @@ -190,9 +217,15 @@ export const knownLibFilesForCompilerOptions = (compilerOptions: CompilerOptions
/**
* Sets up a Map with lib contents by grabbing the necessary files from
* the local copy of typescript via the file system.
*
* The first two args are un-used, but kept around so as to not cause a
* semver major bump for no gain to module users.
*/
export const createDefaultMapFromNodeModules = (compilerOptions: CompilerOptions, ts?: typeof import("typescript"), tsLibDirectory?: string) => {
const tsModule = ts || require("typescript")
export const createDefaultMapFromNodeModules = (
_compilerOptions: CompilerOptions,
_ts?: typeof import("typescript"),
tsLibDirectory?: string
) => {
const path = requirePath()
const fs = requireFS()

Expand All @@ -201,9 +234,11 @@ export const createDefaultMapFromNodeModules = (compilerOptions: CompilerOptions
return fs.readFileSync(path.join(lib, name), "utf8")
}

const libs = knownLibFilesForCompilerOptions(compilerOptions, tsModule)
const libFiles = fs.readdirSync(tsLibDirectory || path.dirname(require.resolve("typescript")))
const knownLibFiles = libFiles.filter(f => f.startsWith("lib.") && f.endsWith(".d.ts"))

const fsMap = new Map<string, string>()
libs.forEach(lib => {
knownLibFiles.forEach(lib => {
fsMap.set("/" + lib, getLib(lib))
})
return fsMap
Expand Down Expand Up @@ -403,7 +438,12 @@ export function createSystem(files: Map<string, string>): System {
* a set of virtual files which are prioritised over the FS versions, then a path to the root of your
* project (basically the folder your node_modules lives)
*/
export function createFSBackedSystem(files: Map<string, string>, _projectRoot: string, ts: TS, tsLibDirectory?: string): System {
export function createFSBackedSystem(
files: Map<string, string>,
_projectRoot: string,
ts: TS,
tsLibDirectory?: string
): System {
// We need to make an isolated folder for the tsconfig, but also need to be able to resolve the
// existing node_modules structures going back through the history
const root = _projectRoot + "/vfs"
Expand Down
5 changes: 5 additions & 0 deletions packages/typescript-vfs/test/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,3 +209,8 @@ it("throws when you request a lib file which isn't in the fsMap", () => {

expect(t).toThrow()
})

it("grabs lib dts files from node_modules", async () => {
const fsMap = createDefaultMapFromNodeModules({})
expect(fsMap.get("/lib.es2015.collection.d.ts")).toBeDefined()
})
2 changes: 1 addition & 1 deletion packages/typescriptlang-org/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"@types/react-helmet": "^5.0.15",
"@typescript/playground": "0.1.0",
"@typescript/sandbox": "0.1.0",
"@typescript/twoslash": "3.2.1",
"@typescript/twoslash": "3.2.2",
"gatsby": "^3.8.1",
"gatsby-link": "3.10.0",
"gatsby-plugin-catch-links": "^3.10.0",
Expand Down
29 changes: 19 additions & 10 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6119,7 +6119,25 @@ __metadata:
languageName: node
linkType: hard

"@typescript/[email protected], @typescript/vfs@workspace:packages/typescript-vfs":
"@typescript/vfs@npm:1.3.4":
version: 1.3.4
resolution: "@typescript/vfs@npm:1.3.4"
dependencies:
debug: ^4.1.1
checksum: 9468126c7f5da5811f65a6a8ccded68ba3ae32e5bff710ff0fc792b9592dd0b64606669703c707a61350d18d0640a60465fd1fdfa19224bf75998061d461cbf5
languageName: node
linkType: hard

"@typescript/vfs@npm:1.4.0":
version: 1.4.0
resolution: "@typescript/vfs@npm:1.4.0"
dependencies:
debug: ^4.1.1
checksum: 67219c9354d142ef8259da172066db043133d8b6cc0a5d862b9867d5950c5f6c9b816e8b29f8901e8cd2572003282fbbec097e53932bc6da01d0fa2e40520014
languageName: node
linkType: hard

"@typescript/vfs@workspace:packages/typescript-vfs":
version: 0.0.0-use.local
resolution: "@typescript/vfs@workspace:packages/typescript-vfs"
dependencies:
Expand All @@ -6136,15 +6154,6 @@ __metadata:
languageName: unknown
linkType: soft

"@typescript/vfs@npm:1.3.4":
version: 1.3.4
resolution: "@typescript/vfs@npm:1.3.4"
dependencies:
debug: ^4.1.1
checksum: 9468126c7f5da5811f65a6a8ccded68ba3ae32e5bff710ff0fc792b9592dd0b64606669703c707a61350d18d0640a60465fd1fdfa19224bf75998061d461cbf5
languageName: node
linkType: hard

"@webassemblyjs/ast@npm:1.11.0":
version: 1.11.0
resolution: "@webassemblyjs/ast@npm:1.11.0"
Expand Down