diff --git a/.travis.yml b/.travis.yml index e90e3ebe48b94c..a30bb170dad8ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,9 @@ cache: - third_party/v8/build/linux/debian_sid_amd64-sysroot/ - third_party/v8/third_party/llvm-build/ install: +- nvm install v8 +- nvm use --delete-prefix v8 +- node -v - |- # OS X: install a private copy of homebrew. if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then diff --git a/BUILD.gn b/BUILD.gn index 8bcbdc4c65ff8c..9a092b0ccdbc1c 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -85,6 +85,7 @@ ts_sources = [ "js/libdeno.ts", "js/main.ts", "js/make_temp_dir.ts", + "js/metrics.ts", "js/mkdir.ts", "js/mock_builtin.js", "js/net.ts", @@ -107,7 +108,6 @@ ts_sources = [ "js/v8_source_maps.ts", "js/write_file.ts", - "js/tsconfig.declarations.json", "tsconfig.json", # Listing package.json and yarn.lock as sources ensures the bundle is rebuilt @@ -246,26 +246,33 @@ executable("snapshot_creator") { configs += [ ":deno_config" ] } -# Generates type declarations for files that need to be included -# in the runtime bundle -run_node("gen_declarations") { +# Generates the core TypeScript type library for deno that will be +# included in the runtime bundle +run_node("deno_runtime_declaration") { out_dir = target_gen_dir sources = ts_sources outputs = [ - "$out_dir/types/globals.d.ts", + "$out_dir/lib/lib.deno_runtime.d.ts", ] deps = [ ":msg_ts", ] args = [ - "./node_modules/typescript/bin/tsc", - "-p", - rebase_path("js/tsconfig.declarations.json", root_build_dir), - "--baseUrl", + rebase_path("node_modules/.bin/ts-node", root_build_dir), + "--project", + rebase_path("tools/ts_library_builder/tsconfig.json"), + rebase_path("tools/ts_library_builder/main.ts", root_build_dir), + "--basePath", + rebase_path(".", root_build_dir), + "--buildPath", rebase_path(root_build_dir, root_build_dir), "--outFile", - rebase_path("$out_dir/types/globals.js", root_build_dir), + rebase_path("$out_dir/lib/lib.deno_runtime.d.ts", root_build_dir), + "--silent", ] + if (is_debug) { + args += [ "--debug" ] + } } run_node("bundle") { @@ -276,7 +283,7 @@ run_node("bundle") { out_dir + "main.js.map", ] deps = [ - ":gen_declarations", + ":deno_runtime_declaration", ":msg_ts", ] args = [ diff --git a/Cargo.toml b/Cargo.toml index 3781cc8521f4cd..b5d4bbb7b748d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,9 @@ dirs = "1.0.3" flatbuffers = { path = "third_party/flatbuffers/rust/flatbuffers/" } futures = "0.1.23" hyper = "0.12.9" -hyper-rustls = "0.14.0" +# The current version of hyper-rustls, 0.14.0, depends on tokio-core, which is +# deprecated. +hyper-rustls = { git = "https://github.com/ctz/hyper-rustls.git" } libc = "0.2.42" log = "0.4.4" rand = "0.4.3" diff --git a/Roadmap.md b/Roadmap.md index ff0f700719fb17..b03f2e21b16191 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -6,7 +6,7 @@ API and Feature requests should be submitted as PRs to this document. ### Implementation of `cat` -#721 +[#721](https://github.com/denoland/deno/issues/721) ```ts import * as deno from "deno"; @@ -20,7 +20,7 @@ for (let i = 1; i < deno.argv.length; i++) { ### TCP Server -#725 +[#725](https://github.com/denoland/deno/issues/725) ```ts import * as deno from "deno"; @@ -199,13 +199,13 @@ compatibility with Node. #### Top-level Await (Not Implemented) -#471 +[#471](https://github.com/denoland/deno/issues/471) This will be put off until at least deno2 Milestone1 is complete. One of the major problems is that top-level await calls are not syntactically valid TypeScript. -#### I/O (Not Implemented) #721 +#### I/O (Not Implemented) [#721](https://github.com/denoland/deno/issues/721) There are many OS constructs that perform I/O: files, sockets, pipes. Deno aims to provide a unified lowest common denominator interface to work with these diff --git a/build_extra/rust/BUILD.gn b/build_extra/rust/BUILD.gn index 763655cfd50294..3d286ba8aa88df 100644 --- a/build_extra/rust/BUILD.gn +++ b/build_extra/rust/BUILD.gn @@ -476,23 +476,6 @@ rust_crate("hyper") { ] } -rust_crate("tokio_core") { - source_root = "$registry_github/tokio-core-0.1.17/src/lib.rs" - extern = [ - ":mio", - ":futures", - ":tokio", - ":tokio_executor", - ":tokio_reactor", - ":tokio_timer", - ":tokio_io", - ":log", - ":iovec", - ":bytes", - ":scoped_tls", - ] -} - rust_crate("h2") { source_root = "$registry_github/h2-0.1.12/src/lib.rs" extern = [ @@ -709,14 +692,14 @@ rust_crate("tokio_current_thread") { } rust_crate("hyper_rustls") { - source_root = "$registry_github/hyper-rustls-0.14.0/src/lib.rs" + source_root = + "$crates/git/checkouts/hyper-rustls-d4ca51501db57c63/2c536d5/src/lib.rs" extern = [ ":ct_logs", ":futures", ":http", ":hyper", ":rustls", - ":tokio_core", ":tokio_io", ":tokio_rustls", ":tokio_tcp", diff --git a/js/assets.ts b/js/assets.ts index c860555c4f6ba5..481c48239d5fb1 100644 --- a/js/assets.ts +++ b/js/assets.ts @@ -7,7 +7,7 @@ // tslint:disable:max-line-length // Generated default library -import globalsDts from "gen/types/globals.d.ts!string"; +import libDts from "gen/lib/lib.deno_runtime.d.ts!string"; // Static libraries import libEs2015Dts from "/third_party/node_modules/typescript/lib/lib.es2015.d.ts!string"; @@ -40,7 +40,6 @@ import libEsnextIntlDts from "/third_party/node_modules/typescript/lib/lib.esnex import libEsnextSymbolDts from "/third_party/node_modules/typescript/lib/lib.esnext.symbol.d.ts!string"; // Static definitions -import flatbuffersDts from "/third_party/node_modules/@types/flatbuffers/index.d.ts!string"; import textEncodingDts from "/third_party/node_modules/@types/text-encoding/index.d.ts!string"; import typescriptDts from "/third_party/node_modules/typescript/lib/typescript.d.ts!string"; // tslint:enable:max-line-length @@ -48,7 +47,7 @@ import typescriptDts from "/third_party/node_modules/typescript/lib/typescript.d // @internal export const assetSourceCode: { [key: string]: string } = { // Generated library - "globals.d.ts": globalsDts, + "lib.deno_runtime.d.ts": libDts, // Static libraries "lib.es2015.collection.d.ts": libEs2015CollectionDts, @@ -81,7 +80,6 @@ export const assetSourceCode: { [key: string]: string } = { "lib.esnext.symbol.d.ts": libEsnextSymbolDts, // Static definitions - "flatbuffers.d.ts": flatbuffersDts, "text-encoding.d.ts": textEncodingDts, "typescript.d.ts": typescriptDts }; diff --git a/js/compiler.ts b/js/compiler.ts index 7b577a8b94f608..c398f181d35505 100644 --- a/js/compiler.ts +++ b/js/compiler.ts @@ -13,6 +13,7 @@ import * as sourceMaps from "./v8_source_maps"; const EOL = "\n"; const ASSETS = "$asset$"; +const LIB_RUNTIME = "lib.deno_runtime.d.ts"; // tslint:disable:no-any type AmdCallback = (...args: any[]) => void; @@ -619,7 +620,7 @@ export class DenoCompiler getDefaultLibFileName(): string { this._log("getDefaultLibFileName()"); - const moduleSpecifier = "globals.d.ts"; + const moduleSpecifier = LIB_RUNTIME; const moduleMetaData = this.resolveModule(moduleSpecifier, ASSETS); return moduleMetaData.fileName; } @@ -649,8 +650,8 @@ export class DenoCompiler return moduleNames.map(name => { let resolvedFileName; if (name === "deno") { - // builtin modules are part of `globals.d.ts` - resolvedFileName = this._resolveModuleName("globals.d.ts", ASSETS); + // builtin modules are part of the runtime lib + resolvedFileName = this._resolveModuleName(LIB_RUNTIME, ASSETS); } else if (name === "typescript") { resolvedFileName = this._resolveModuleName("typescript.d.ts", ASSETS); } else { diff --git a/js/compiler_test.ts b/js/compiler_test.ts index dcb8c128520bd7..f05a96e52baed1 100644 --- a/js/compiler_test.ts +++ b/js/compiler_test.ts @@ -546,7 +546,10 @@ test(function compilerGetCurrentDirectory() { test(function compilerGetDefaultLibFileName() { setup(); - assertEqual(compilerInstance.getDefaultLibFileName(), "$asset$/globals.d.ts"); + assertEqual( + compilerInstance.getDefaultLibFileName(), + "$asset$/lib.deno_runtime.d.ts" + ); teardown(); }); @@ -572,7 +575,7 @@ test(function compilerFileExists() { "/root/project" ); assert(compilerInstance.fileExists(moduleMetaData.fileName)); - assert(compilerInstance.fileExists("$asset$/globals.d.ts")); + assert(compilerInstance.fileExists("$asset$/lib.deno_runtime.d.ts")); assertEqual( compilerInstance.fileExists("/root/project/unknown-module.ts"), false @@ -590,7 +593,7 @@ test(function compilerResolveModuleNames() { const fixtures: Array<[string, boolean]> = [ ["/root/project/foo/bar.ts", false], ["/root/project/foo/baz.ts", false], - ["$asset$/globals.d.ts", true] + ["$asset$/lib.deno_runtime.d.ts", true] ]; for (let i = 0; i < results.length; i++) { const result = results[i]; diff --git a/js/deno.ts b/js/deno.ts index a7350e03e7ef6d..2125445411432d 100644 --- a/js/deno.ts +++ b/js/deno.ts @@ -35,6 +35,7 @@ export { trace } from "./trace"; export { truncateSync, truncate } from "./truncate"; export { FileInfo } from "./file_info"; export { connect, dial, listen, Listener, Conn } from "./net"; +export { metrics } from "./metrics"; export const args: string[] = []; // Provide the compiler API in an obfuscated way diff --git a/js/files.ts b/js/files.ts index a23ce474503674..c3722a83ffed76 100644 --- a/js/files.ts +++ b/js/files.ts @@ -7,18 +7,18 @@ import { assert } from "./util"; import { flatbuffers } from "flatbuffers"; export class File implements Reader, Writer, Closer { - constructor(readonly fd: number) {} + constructor(readonly rid: number) {} write(p: ArrayBufferView): Promise { - return write(this.fd, p); + return write(this.rid, p); } read(p: ArrayBufferView): Promise { - return read(this.fd, p); + return read(this.rid, p); } close(): void { - close(this.fd); + close(this.rid); } } @@ -47,17 +47,17 @@ export async function open( assert(msg.Any.OpenRes === baseRes!.innerType()); const res = new msg.OpenRes(); assert(baseRes!.inner(res) != null); - const fd = res.rid(); - return new File(fd); + const rid = res.rid(); + return new File(rid); } export async function read( - fd: number, + rid: number, p: ArrayBufferView ): Promise { const builder = new flatbuffers.Builder(); msg.Read.startRead(builder); - msg.Read.addRid(builder, fd); + msg.Read.addRid(builder, rid); const inner = msg.Read.endRead(builder); const baseRes = await dispatch.sendAsync(builder, msg.Any.Read, inner, p); assert(baseRes != null); @@ -67,10 +67,10 @@ export async function read( return { nread: res.nread(), eof: res.eof() }; } -export async function write(fd: number, p: ArrayBufferView): Promise { +export async function write(rid: number, p: ArrayBufferView): Promise { const builder = new flatbuffers.Builder(); msg.Write.startWrite(builder); - msg.Write.addRid(builder, fd); + msg.Write.addRid(builder, rid); const inner = msg.Write.endWrite(builder); const baseRes = await dispatch.sendAsync(builder, msg.Any.Write, inner, p); assert(baseRes != null); @@ -80,10 +80,10 @@ export async function write(fd: number, p: ArrayBufferView): Promise { return res.nbyte(); } -export function close(fd: number): void { +export function close(rid: number): void { const builder = new flatbuffers.Builder(); msg.Close.startClose(builder); - msg.Close.addRid(builder, fd); + msg.Close.addRid(builder, rid); const inner = msg.Close.endClose(builder); dispatch.sendSync(builder, msg.Any.Close, inner); } diff --git a/js/files_test.ts b/js/files_test.ts index 82af10aa2c8355..835e06c844ef7b 100644 --- a/js/files_test.ts +++ b/js/files_test.ts @@ -4,15 +4,15 @@ import * as deno from "deno"; import { test, assert, assertEqual } from "./test_util.ts"; test(function filesStdioFileDescriptors() { - assertEqual(deno.stdin.fd, 0); - assertEqual(deno.stdout.fd, 1); - assertEqual(deno.stderr.fd, 2); + assertEqual(deno.stdin.rid, 0); + assertEqual(deno.stdout.rid, 1); + assertEqual(deno.stderr.rid, 2); }); test(async function filesCopyToStdout() { const filename = "package.json"; const file = await deno.open(filename); - assert(file.fd > 2); + assert(file.rid > 2); const bytesWritten = await deno.copy(deno.stdout, file); const fileSize = deno.statSync(filename).len; assertEqual(bytesWritten, fileSize); diff --git a/js/globals.ts b/js/globals.ts index 2e6b24305e4fd7..11a1726333bc93 100644 --- a/js/globals.ts +++ b/js/globals.ts @@ -1,55 +1,22 @@ // Copyright 2018 the Deno authors. All rights reserved. MIT license. -import { Console } from "./console"; -import * as timers from "./timers"; -import * as textEncoding from "./text_encoding"; +import * as blob from "./blob"; +import * as console from "./console"; import * as fetch_ from "./fetch"; -import { libdeno } from "./libdeno"; import { globalEval } from "./global_eval"; -import { DenoHeaders } from "./fetch"; -import { DenoBlob } from "./blob"; - -declare global { - interface Window { - define: Readonly; - - clearTimeout: typeof clearTimeout; - clearInterval: typeof clearInterval; - setTimeout: typeof setTimeout; - setInterval: typeof setInterval; - - console: typeof console; - window: typeof window; - - fetch: typeof fetch; - - TextEncoder: typeof TextEncoder; - TextDecoder: typeof TextDecoder; - atob: typeof atob; - btoa: typeof btoa; +import { libdeno } from "./libdeno"; +import * as textEncoding from "./text_encoding"; +import * as timers from "./timers"; - Headers: typeof Headers; - Blob: typeof Blob; - } +// During the build process, augmentations to the variable `window` in this +// file are tracked and created as part of default library that is built into +// deno, we only need to declare the enough to compile deno. - const clearTimeout: typeof timers.clearTimer; - const clearInterval: typeof timers.clearTimer; +declare global { + const console: console.Console; const setTimeout: typeof timers.setTimeout; - const setInterval: typeof timers.setInterval; - - const console: Console; - const window: Window; - - const fetch: typeof fetch_.fetch; - - // tslint:disable:variable-name + // tslint:disable-next-line:variable-name const TextEncoder: typeof textEncoding.TextEncoder; - const TextDecoder: typeof textEncoding.TextDecoder; - const atob: typeof textEncoding.atob; - const btoa: typeof textEncoding.btoa; - const Headers: typeof DenoHeaders; - const Blob: typeof DenoBlob; - // tslint:enable:variable-name } // A reference to the global object. @@ -61,7 +28,7 @@ window.setInterval = timers.setInterval; window.clearTimeout = timers.clearTimer; window.clearInterval = timers.clearTimer; -window.console = new Console(libdeno.print); +window.console = new console.Console(libdeno.print); window.TextEncoder = textEncoding.TextEncoder; window.TextDecoder = textEncoding.TextDecoder; window.atob = textEncoding.atob; @@ -69,5 +36,5 @@ window.btoa = textEncoding.btoa; window.fetch = fetch_.fetch; -window.Headers = DenoHeaders; -window.Blob = DenoBlob; +window.Headers = fetch_.DenoHeaders; +window.Blob = blob.DenoBlob; diff --git a/js/io.ts b/js/io.ts index 710722f42e2ca5..77ef790732cdcf 100644 --- a/js/io.ts +++ b/js/io.ts @@ -101,7 +101,7 @@ export interface ReadWriteSeeker extends Reader, Writer, Seeker {} // https://golang.org/pkg/io/#Copy export async function copy(dst: Writer, src: Reader): Promise { let n = 0; - const b = new Uint8Array(1024); + const b = new Uint8Array(32*1024); let gotEOF = false; while (gotEOF === false) { const result = await src.read(b); diff --git a/js/main.ts b/js/main.ts index 1941327975c0ee..d34def274cdd28 100644 --- a/js/main.ts +++ b/js/main.ts @@ -48,6 +48,14 @@ export default function denoMain() { setLogDebug(startResMsg.debugFlag()); + // handle `--types` + if (startResMsg.typesFlag()) { + const defaultLibFileName = compiler.getDefaultLibFileName(); + const defaultLibModule = compiler.resolveModule(defaultLibFileName, ""); + console.log(defaultLibModule.sourceCode); + os.exit(0); + } + const cwd = startResMsg.cwd(); log("cwd", cwd); @@ -64,9 +72,9 @@ export default function denoMain() { console.log("No input script specified."); // os.exit(1); } - else { - const printDeps = startResMsg.depsFlag(); - if (printDeps) { + + // handle `--deps` + if (startResMsg.depsFlag()) { for (const dep of compiler.getModuleDependencies(inputFn, `${cwd}/`)) { console.log(dep); } diff --git a/js/metrics.ts b/js/metrics.ts new file mode 100644 index 00000000000000..d76b781db6f7cb --- /dev/null +++ b/js/metrics.ts @@ -0,0 +1,39 @@ +// Copyright 2018 the Deno authors. All rights reserved. MIT license. +import * as msg from "gen/msg_generated"; +import { flatbuffers } from "flatbuffers"; +import { assert } from "./util"; +import * as dispatch from "./dispatch"; + +interface Metrics { + opsDispatched: number; + opsCompleted: number; + bytesSentControl: number; + bytesSentData: number; + bytesReceived: number; +} + +export function metrics(): Metrics { + return res(dispatch.sendSync(...req())); +} + +function req(): [flatbuffers.Builder, msg.Any, flatbuffers.Offset] { + const builder = new flatbuffers.Builder(); + msg.Metrics.startMetrics(builder); + const inner = msg.Metrics.endMetrics(builder); + return [builder, msg.Any.Metrics, inner]; +} + +function res(baseRes: null | msg.Base): Metrics { + assert(baseRes !== null); + assert(msg.Any.MetricsRes === baseRes!.innerType()); + const res = new msg.MetricsRes(); + assert(baseRes!.inner(res) !== null); + + return { + opsDispatched: res.opsDispatched().toFloat64(), + opsCompleted: res.opsCompleted().toFloat64(), + bytesSentControl: res.bytesSentControl().toFloat64(), + bytesSentData: res.bytesSentData().toFloat64(), + bytesReceived: res.bytesReceived().toFloat64() + }; +} diff --git a/js/metrics_test.ts b/js/metrics_test.ts new file mode 100644 index 00000000000000..6954ae2ce5cbc0 --- /dev/null +++ b/js/metrics_test.ts @@ -0,0 +1,24 @@ +// Copyright 2018 the Deno authors. All rights reserved. MIT license. +import { test, assert } from "./test_util.ts"; +import * as deno from "deno"; + +test(function metrics() { + const m1 = deno.metrics(); + assert(m1.opsDispatched > 0); + assert(m1.opsCompleted > 0); + assert(m1.bytesSentControl > 0); + assert(m1.bytesSentData >= 0); + assert(m1.bytesReceived > 0); + + // Write to stdout to ensure a "data" message gets sent instead of just + // control messages. + const dataMsg = new Uint8Array([41, 42, 43]); + deno.stdout.write(dataMsg); + + const m2 = deno.metrics(); + assert(m2.opsDispatched > m1.opsDispatched); + assert(m2.opsCompleted > m1.opsCompleted); + assert(m2.bytesSentControl > m1.bytesSentControl); + assert(m2.bytesSentData >= m1.bytesSentData + dataMsg.byteLength); + assert(m2.bytesReceived > m1.bytesReceived); +}); diff --git a/js/net.ts b/js/net.ts index 1258a0ff195985..da8a0303e1038f 100644 --- a/js/net.ts +++ b/js/net.ts @@ -28,12 +28,12 @@ export interface Listener { } class ListenerImpl implements Listener { - constructor(readonly fd: number) {} + constructor(readonly rid: number) {} async accept(): Promise { const builder = new flatbuffers.Builder(); msg.Accept.startAccept(builder); - msg.Accept.addRid(builder, this.fd); + msg.Accept.addRid(builder, this.rid); const inner = msg.Accept.endAccept(builder); const baseRes = await dispatch.sendAsync(builder, msg.Any.Accept, inner); assert(baseRes != null); @@ -44,7 +44,7 @@ class ListenerImpl implements Listener { } close(): void { - close(this.fd); + close(this.rid); } addr(): Addr { @@ -61,35 +61,35 @@ export interface Conn extends Reader, Writer, Closer { class ConnImpl implements Conn { constructor( - readonly fd: number, + readonly rid: number, readonly remoteAddr: string, readonly localAddr: string ) {} write(p: ArrayBufferView): Promise { - return write(this.fd, p); + return write(this.rid, p); } read(p: ArrayBufferView): Promise { - return read(this.fd, p); + return read(this.rid, p); } close(): void { - close(this.fd); + close(this.rid); } /** closeRead shuts down (shutdown(2)) the reading side of the TCP connection. * Most callers should just use close(). */ closeRead(): void { - shutdown(this.fd, ShutdownMode.Read); + shutdown(this.rid, ShutdownMode.Read); } /** closeWrite shuts down (shutdown(2)) the writing side of the TCP * connection. Most callers should just use close(). */ closeWrite(): void { - shutdown(this.fd, ShutdownMode.Write); + shutdown(this.rid, ShutdownMode.Write); } } @@ -101,10 +101,10 @@ enum ShutdownMode { ReadWrite // unused } -function shutdown(fd: number, how: ShutdownMode) { +function shutdown(rid: number, how: ShutdownMode) { const builder = new flatbuffers.Builder(); msg.Shutdown.startShutdown(builder); - msg.Shutdown.addRid(builder, fd); + msg.Shutdown.addRid(builder, rid); msg.Shutdown.addHow(builder, how); const inner = msg.Shutdown.endShutdown(builder); const baseRes = dispatch.sendSync(builder, msg.Any.Shutdown, inner); diff --git a/js/testing/testing.ts b/js/testing/testing.ts index ff91ddebc5c22b..437cb0f38fe4a8 100644 --- a/js/testing/testing.ts +++ b/js/testing/testing.ts @@ -13,7 +13,7 @@ limitations under the License. */ -export { assert, assertEqual, equal } from "./util.ts"; +export { assert, assertEqual, equal } from "./util"; export type TestFunction = () => void | Promise; diff --git a/js/timers_test.ts b/js/timers_test.ts index 8d5245f11bff95..78f6610c48ab03 100644 --- a/js/timers_test.ts +++ b/js/timers_test.ts @@ -32,25 +32,21 @@ test(async function timeoutSuccess() { }); test(async function timeoutArgs() { + const { promise, resolve } = deferred(); const arg = 1; - await new Promise((resolve, reject) => { - setTimeout( - (a, b, c) => { - try { - assertEqual(a, arg); - assertEqual(b, arg.toString()); - assertEqual(c, [arg]); - resolve(); - } catch (e) { - reject(e); - } - }, - 10, - arg, - arg.toString(), - [arg] - ); - }); + setTimeout( + (a, b, c) => { + assertEqual(a, arg); + assertEqual(b, arg.toString()); + assertEqual(c, [arg]); + resolve(); + }, + 10, + arg, + arg.toString(), + [arg] + ); + await promise; }); test(async function timeoutCancelSuccess() { @@ -151,16 +147,8 @@ test(async function intervalOrdering() { clearTimeout(timers[i]); } } - await new Promise((resolve, reject) => { - setTimeout(() => { - try { - assertEqual(timeouts, 1); - resolve(); - } catch (e) { - reject(e); - } - }, 100); - }); + await waitForMs(100); + assertEqual(timeouts, 1); }); test(async function intervalCancelInvalidSilentFail() { diff --git a/js/tsconfig.declarations.json b/js/tsconfig.declarations.json deleted file mode 100644 index 6371182b6576bf..00000000000000 --- a/js/tsconfig.declarations.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - // This configuration file provides the tsc configuration for generating - // definitions for the runtime, which are then inlined via the `js/assets.ts` - // module into the bundle to be available for type checking at runtime - // See also gen_declarations in //BUILD.gn - "extends": "../tsconfig.json", - "compilerOptions": { - "declaration": true, - "emitDeclarationOnly": true, - "module": "amd", - "removeComments": false, - "stripInternal": true - }, - "files": [ - "../node_modules/typescript/lib/lib.esnext.d.ts", - "./deno.ts", - "./globals.ts" - ] -} diff --git a/js/unit_tests.ts b/js/unit_tests.ts index ca152ad3919eee..24fdac823425a4 100644 --- a/js/unit_tests.ts +++ b/js/unit_tests.ts @@ -25,3 +25,4 @@ import "./trace_test.ts"; import "./truncate_test.ts"; import "./v8_source_maps_test.ts"; import "../website/app_test.js"; +import "./metrics_test.ts"; diff --git a/package.json b/package.json index fd2b56c4e922e4..c25c94ee38456d 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "devDependencies": { "@types/base64-js": "^1.2.5", "@types/flatbuffers": "^1.9.0", + "@types/prettier": "^1.13.2", "@types/source-map-support": "^0.4.1", "@types/text-encoding": "0.0.33", "base64-js": "^1.3.0", @@ -20,6 +21,8 @@ "rollup-pluginutils": "^2.3.0", "source-map-support": "^0.5.6", "text-encoding": "0.6.4", + "ts-node": "^7.0.1", + "ts-simple-ast": "^16.0.4", "tslint": "^5.10.0", "tslint-eslint-rules": "^5.3.1", "tslint-no-circular-imports": "^0.5.0", diff --git a/rollup.config.js b/rollup.config.js index 4cb3197f5b74cd..55f832585585e1 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -26,12 +26,6 @@ const tsconfigOverride = { } }; -// this is a preamble for the `globals.d.ts` file to allow it to be the default -// lib for deno. -const libPreamble = `/// -/// -`; - // this is a rollup plugin which will look for imports ending with `!string` and resolve // them with a module that will inline the contents of the file as a string. Needed to // support `js/assets.ts`. @@ -70,9 +64,7 @@ function strings({ include, exclude } = {}) { transform(code, id) { if (filter(id)) { return { - code: `export default ${JSON.stringify( - id.endsWith("globals.d.ts") ? libPreamble + code : code - )};`, + code: `export default ${JSON.stringify(code)};`, map: { mappings: "" } }; } diff --git a/src/deno_dir.rs b/src/deno_dir.rs index 1e369b40f8fe2c..e8cc86b1863fc9 100644 --- a/src/deno_dir.rs +++ b/src/deno_dir.rs @@ -4,7 +4,7 @@ use errors::DenoError; use errors::DenoResult; use errors::ErrorKind; use fs as deno_fs; -use http; +use http_util; use ring; use std; use std::fmt::Write; @@ -114,7 +114,7 @@ impl DenoDir { let src = if self.reload || !p.exists() { println!("Downloading {}", module_name); - let source = http::fetch_sync_string(module_name)?; + let source = http_util::fetch_sync_string(module_name)?; match p.parent() { Some(ref parent) => fs::create_dir_all(parent), None => Ok(()), diff --git a/src/errors.rs b/src/errors.rs index ab6504e64ca653..0f5320348f5238 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -150,10 +150,7 @@ impl From for DenoError { } pub fn bad_resource() -> DenoError { - new( - ErrorKind::BadFileDescriptor, // TODO Rename to BadResource - String::from("bad resource id"), - ) + new(ErrorKind::BadResource, String::from("bad resource id")) } pub fn permission_denied() -> DenoError { diff --git a/src/flags.rs b/src/flags.rs index 82b7f528d30688..e4e3c28dfecd22 100644 --- a/src/flags.rs +++ b/src/flags.rs @@ -26,6 +26,7 @@ pub struct DenoFlags { pub allow_net: bool, pub allow_env: bool, pub deps_flag: bool, + pub types_flag: bool, } pub fn process(flags: &DenoFlags) { @@ -58,7 +59,8 @@ pub fn print_usage() { -D or --log-debug Log debug output. -h or --help Print this message. --v8-options Print V8 command line options. ---deps Print module dependencies." +--deps Print module dependencies. +--types Print runtime TypeScript declarations." ); } @@ -82,6 +84,7 @@ pub fn set_flags(args: Vec) -> (DenoFlags, Vec) { "--allow-net" => flags.allow_net = true, "--allow-env" => flags.allow_env = true, "--deps" => flags.deps_flag = true, + "--types" => flags.types_flag = true, "--" => break, _ => unimplemented!(), } @@ -165,6 +168,19 @@ fn test_set_flags_4() { ); } +#[test] +fn test_set_flags_5() { + let (flags, rest) = set_flags(svec!["deno", "--types"]); + assert_eq!(rest, svec!["deno"]); + assert_eq!( + flags, + DenoFlags { + types_flag: true, + ..DenoFlags::default() + } + ) +} + // Returns args passed to V8, followed by args passed to JS fn v8_set_flags_preprocess(args: Vec) -> (Vec, Vec) { let mut rest = vec![]; @@ -179,7 +195,8 @@ fn v8_set_flags_preprocess(args: Vec) -> (Vec, Vec) { } true - }).collect(); + }) + .collect(); // Replace args being sent to V8 for idx in 0..args.len() { @@ -248,6 +265,7 @@ pub fn v8_set_flags(args: Vec) -> Vec { let cstr = CStr::from_ptr(*ptr as *const i8); let slice = cstr.to_str().unwrap(); slice.to_string() - }).chain(rest.into_iter()) + }) + .chain(rest.into_iter()) .collect() } diff --git a/src/http.rs b/src/http_util.rs similarity index 100% rename from src/http.rs rename to src/http_util.rs diff --git a/src/isolate.rs b/src/isolate.rs index dc44b11e2ca460..40dff5ed254b3a 100644 --- a/src/isolate.rs +++ b/src/isolate.rs @@ -55,6 +55,7 @@ pub struct IsolateState { pub argv: Vec, pub flags: flags::DenoFlags, tx: Mutex>>, + pub metrics: Mutex, } impl IsolateState { @@ -66,6 +67,32 @@ impl IsolateState { let tx = maybe_tx.unwrap(); tx.send((req_id, buf)).expect("tx.send error"); } + + fn metrics_op_dispatched( + &self, + bytes_sent_control: u64, + bytes_sent_data: u64, + ) { + let mut metrics = self.metrics.lock().unwrap(); + metrics.ops_dispatched += 1; + metrics.bytes_sent_control += bytes_sent_control; + metrics.bytes_sent_data += bytes_sent_data; + } + + fn metrics_op_completed(&self, bytes_received: u64) { + let mut metrics = self.metrics.lock().unwrap(); + metrics.ops_completed += 1; + metrics.bytes_received += bytes_received; + } +} + +#[derive(Default)] +pub struct Metrics { + pub ops_dispatched: u64, + pub ops_completed: u64, + pub bytes_sent_control: u64, + pub bytes_sent_data: u64, + pub bytes_received: u64, } static DENO_INIT: std::sync::Once = std::sync::ONCE_INIT; @@ -92,6 +119,7 @@ impl Isolate { argv: argv_rest, flags, tx: Mutex::new(Some(tx)), + metrics: Mutex::new(Metrics::default()), }), } } @@ -129,6 +157,7 @@ impl Isolate { } pub fn respond(&mut self, req_id: i32, buf: Buf) { + self.state.metrics_op_completed(buf.len() as u64); // TODO(zero-copy) Use Buf::leak(buf) to leak the heap allocated buf. And // don't do the memcpy in ImportBuf() (in libdeno/binding.cc) unsafe { @@ -171,31 +200,11 @@ impl Isolate { pub fn event_loop(&mut self) { // Main thread event loop. while !self.is_idle() { - // Ideally, mpsc::Receiver would have a receive method that takes a optional - // timeout. But it doesn't so we need all this duplicate code. - match self.timeout_due { - Some(due) => { - // Subtracting two Instants causes a panic if the resulting duration - // would become negative. Avoid this. - let now = Instant::now(); - let timeout = if due > now { - due - now - } else { - Duration::new(0, 0) - }; - // TODO: use recv_deadline() instead of recv_timeout() when this - // feature becomes stable/available. - match self.rx.recv_timeout(timeout) { - Ok((req_id, buf)) => self.complete_op(req_id, buf), - Err(mpsc::RecvTimeoutError::Timeout) => self.timeout(), - Err(e) => panic!("mpsc::Receiver::recv_timeout() failed: {:?}", e), - } - } - None => match self.rx.recv() { - Ok((req_id, buf)) => self.complete_op(req_id, buf), - Err(e) => panic!("mpsc::Receiver::recv() failed: {:?}", e), - }, - }; + match recv_deadline(&self.rx, self.timeout_due) { + Ok((req_id, buf)) => self.complete_op(req_id, buf), + Err(mpsc::RecvTimeoutError::Timeout) => self.timeout(), + Err(e) => panic!("recv_deadline() failed: {:?}", e), + } } } @@ -241,6 +250,10 @@ extern "C" fn pre_dispatch( control_buf: libdeno::deno_buf, data_buf: libdeno::deno_buf, ) { + // for metrics + let bytes_sent_control = control_buf.data_len as u64; + let bytes_sent_data = data_buf.data_len as u64; + // control_buf is only valid for the lifetime of this call, thus is // interpretted as a slice. let control_slice = unsafe { @@ -260,16 +273,21 @@ extern "C" fn pre_dispatch( let dispatch = isolate.dispatch; let (is_sync, op) = dispatch(isolate, control_slice, data_slice); + isolate + .state + .metrics_op_dispatched(bytes_sent_control, bytes_sent_data); + if is_sync { // Execute op synchronously. let buf = tokio_util::block_on(op).unwrap(); - if buf.len() != 0 { + let buf_size = buf.len(); + if buf_size != 0 { // Set the synchronous response, the value returned from isolate.send(). isolate.respond(req_id, buf); } } else { // Execute op asynchronously. - let state = isolate.state.clone(); + let state = Arc::clone(&isolate.state); // TODO Ideally Tokio would could tell us how many tasks are executing, but // it cannot currently. Therefore we track top-level promises/tasks @@ -285,6 +303,28 @@ extern "C" fn pre_dispatch( } } +fn recv_deadline( + rx: &mpsc::Receiver, + maybe_due: Option, +) -> Result { + match maybe_due { + None => rx.recv().map_err(|e| e.into()), + Some(due) => { + // Subtracting two Instants causes a panic if the resulting duration + // would become negative. Avoid this. + let now = Instant::now(); + let timeout = if due > now { + due - now + } else { + Duration::new(0, 0) + }; + // TODO: use recv_deadline() instead of recv_timeout() when this + // feature becomes stable/available. + rx.recv_timeout(timeout) + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -308,7 +348,8 @@ mod tests { throw Error("assert error"); } "#, - ).expect("execute error"); + ) + .expect("execute error"); isolate.event_loop(); }); } @@ -328,4 +369,115 @@ mod tests { let op = Box::new(futures::future::ok(control)); (true, op) } + + #[test] + fn test_metrics_sync() { + let argv = vec![String::from("./deno"), String::from("hello.js")]; + let mut isolate = Isolate::new(argv, metrics_dispatch_sync); + tokio_util::init(|| { + // Verify that metrics have been properly initialized. + { + let metrics = isolate.state.metrics.lock().unwrap(); + assert_eq!(metrics.ops_dispatched, 0); + assert_eq!(metrics.ops_completed, 0); + assert_eq!(metrics.bytes_sent_control, 0); + assert_eq!(metrics.bytes_sent_data, 0); + assert_eq!(metrics.bytes_received, 0); + } + + isolate + .execute( + "y.js", + r#" + const control = new Uint8Array([4, 5, 6]); + const data = new Uint8Array([42, 43, 44, 45, 46]); + libdeno.send(control, data); + "#, + ) + .expect("execute error"); + isolate.event_loop(); + let metrics = isolate.state.metrics.lock().unwrap(); + assert_eq!(metrics.ops_dispatched, 1); + assert_eq!(metrics.ops_completed, 1); + assert_eq!(metrics.bytes_sent_control, 3); + assert_eq!(metrics.bytes_sent_data, 5); + assert_eq!(metrics.bytes_received, 4); + }); + } + + #[test] + fn test_metrics_async() { + let argv = vec![String::from("./deno"), String::from("hello.js")]; + let mut isolate = Isolate::new(argv, metrics_dispatch_async); + tokio_util::init(|| { + // Verify that metrics have been properly initialized. + { + let metrics = isolate.state.metrics.lock().unwrap(); + assert_eq!(metrics.ops_dispatched, 0); + assert_eq!(metrics.ops_completed, 0); + assert_eq!(metrics.bytes_sent_control, 0); + assert_eq!(metrics.bytes_sent_data, 0); + assert_eq!(metrics.bytes_received, 0); + } + + isolate + .execute( + "y.js", + r#" + const control = new Uint8Array([4, 5, 6]); + const data = new Uint8Array([42, 43, 44, 45, 46]); + let r = libdeno.send(control, data); + if (r != null) throw Error("expected null"); + "#, + ) + .expect("execute error"); + + // Make sure relevant metrics are updated before task is executed. + { + let metrics = isolate.state.metrics.lock().unwrap(); + assert_eq!(metrics.ops_dispatched, 1); + assert_eq!(metrics.bytes_sent_control, 3); + assert_eq!(metrics.bytes_sent_data, 5); + // Note we cannot check ops_completed nor bytes_received because that + // would be a race condition. It might be nice to have use a oneshot + // with metrics_dispatch_async() to properly validate them. + } + + isolate.event_loop(); + + // Make sure relevant metrics are updated after task is executed. + { + let metrics = isolate.state.metrics.lock().unwrap(); + assert_eq!(metrics.ops_dispatched, 1); + assert_eq!(metrics.ops_completed, 1); + assert_eq!(metrics.bytes_sent_control, 3); + assert_eq!(metrics.bytes_sent_data, 5); + assert_eq!(metrics.bytes_received, 4); + } + }); + } + + fn metrics_dispatch_sync( + _isolate: &mut Isolate, + _control: &[u8], + _data: &'static mut [u8], + ) -> (bool, Box) { + // Send back some sync response + let vec: Vec = vec![1, 2, 3, 4]; + let control = vec.into_boxed_slice(); + let op = Box::new(futures::future::ok(control)); + (true, op) + } + + fn metrics_dispatch_async( + _isolate: &mut Isolate, + _control: &[u8], + _data: &'static mut [u8], + ) -> (bool, Box) { + // Send back some sync response + let vec: Vec = vec![1, 2, 3, 4]; + let control = vec.into_boxed_slice(); + let op = Box::new(futures::future::ok(control)); + (false, op) + } } diff --git a/src/main.rs b/src/main.rs index 98a2d87dc4de1a..ae1612378c20a9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,7 +32,7 @@ mod deno_dir; mod errors; mod flags; mod fs; -mod http; +mod http_util; mod isolate; mod libdeno; pub mod ops; diff --git a/src/msg.fbs b/src/msg.fbs index 16d10cdff0b638..43fde4d688f6f2 100644 --- a/src/msg.fbs +++ b/src/msg.fbs @@ -41,6 +41,8 @@ union Any { Accept, Dial, NewConn, + Metrics, + MetricsRes, } enum ErrorKind: byte { @@ -67,7 +69,7 @@ enum ErrorKind: byte { Other, UnexpectedEof, - BadFileDescriptor, + BadResource, // url errors @@ -109,6 +111,7 @@ table StartRes { debug_flag: bool; deps_flag: bool; recompile_flag: bool; + types_flag: bool; } table CodeFetch { @@ -321,4 +324,14 @@ table NewConn { local_addr: string; } +table Metrics {} + +table MetricsRes { + ops_dispatched: uint64; + ops_completed: uint64; + bytes_sent_control: uint64; + bytes_sent_data: uint64; + bytes_received: uint64; +} + root_type Base; diff --git a/src/ops.rs b/src/ops.rs index f590a3bb562f5c..a58532bafc0db2 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -101,6 +101,7 @@ pub fn dispatch( msg::Any::Listen => op_listen, msg::Any::Accept => op_accept, msg::Any::Dial => op_dial, + msg::Any::Metrics => op_metrics, _ => panic!(format!( "Unhandled message {}", msg::enum_name_any(inner_type) @@ -184,6 +185,7 @@ fn op_start( argv: Some(argv_off), debug_flag: state.flags.log_debug, recompile_flag: state.flags.recompile, + types_flag: state.flags.types_flag, ..Default::default() }, ); @@ -341,7 +343,8 @@ fn op_env( ..Default::default() }, ) - }).collect(); + }) + .collect(); let tables = builder.create_vector(&vars); let inner = msg::EnvironRes::create( builder, @@ -465,7 +468,7 @@ where // fn blocking(is_sync: bool, f: F) -> Box // where F: FnOnce() -> DenoResult macro_rules! blocking { - ($is_sync:expr,$fn:expr) => { + ($is_sync:expr, $fn:expr) => { if $is_sync { // If synchronous, execute the function immediately on the main thread. Box::new(futures::future::result($fn())) @@ -609,10 +612,7 @@ fn op_shutdown( let rid = inner.rid(); let how = inner.how(); match resources::lookup(rid) { - None => odd_future(errors::new( - errors::ErrorKind::BadFileDescriptor, - String::from("Bad File Descriptor"), - )), + None => odd_future(errors::bad_resource()), Some(mut resource) => { let shutdown_mode = match how { 0 => Shutdown::Read, @@ -893,7 +893,8 @@ fn op_read_dir( ..Default::default() }, ) - }).collect(); + }) + .collect(); let entries = builder.create_vector(&entries); let inner = msg::ReadDirRes::create( @@ -1156,3 +1157,36 @@ fn op_dial( .and_then(move |tcp_stream| new_conn(cmd_id, tcp_stream)); Box::new(op) } + +fn op_metrics( + state: Arc, + base: &msg::Base, + data: &'static mut [u8], +) -> Box { + assert_eq!(data.len(), 0); + let cmd_id = base.cmd_id(); + + let metrics = state.metrics.lock().unwrap(); + + let builder = &mut FlatBufferBuilder::new(); + let inner = msg::MetricsRes::create( + builder, + &msg::MetricsResArgs { + ops_dispatched: metrics.ops_dispatched, + ops_completed: metrics.ops_completed, + bytes_sent_control: metrics.bytes_sent_control, + bytes_sent_data: metrics.bytes_sent_data, + bytes_received: metrics.bytes_received, + ..Default::default() + }, + ); + ok_future(serialize_response( + cmd_id, + builder, + msg::BaseArgs { + inner: Some(inner.as_union_value()), + inner_type: msg::Any::MetricsRes, + ..Default::default() + }, + )) +} diff --git a/tests/echo_server.ts b/tests/echo_server.ts new file mode 100644 index 00000000000000..7761c1ad798fe3 --- /dev/null +++ b/tests/echo_server.ts @@ -0,0 +1,9 @@ +import { args, listen, copy } from "deno"; +const addr = args[1] || "127.0.0.1:4544"; +const listener = listen("tcp", addr); +console.log("listening on", addr); +listener.accept().then(async conn => { + await copy(conn, conn); + conn.close(); + listener.close(); +}); diff --git a/tests/error_003_typescript.ts.out b/tests/error_003_typescript.ts.out index 1423ae1d42ea4f..438b65609df8bc 100644 --- a/tests/error_003_typescript.ts.out +++ b/tests/error_003_typescript.ts.out @@ -3,8 +3,8 @@ [WILDCARD][0m consol.log("hello world!");   ~~~~~~ - $asset$/globals.d.tsILDCARD] -[WILDCARD]const console: Console; + $asset$/lib.deno_runtime.d.tsILDCARD] +[WILDCARD]const console: console.Console; [WILDCARD]~~~~~~~ [WILDCARD]'console' is declared here. diff --git a/third_party b/third_party index 30c059313d7b54..a133fa714b960d 160000 --- a/third_party +++ b/third_party @@ -1 +1 @@ -Subproject commit 30c059313d7b5440ca91dc06619f30958369088d +Subproject commit a133fa714b960d8f88c55188ccc1a41882961e6e diff --git a/tools/benchmark.py b/tools/benchmark.py index ec233afdaf3c4b..4422764aa27d49 100755 --- a/tools/benchmark.py +++ b/tools/benchmark.py @@ -13,6 +13,7 @@ from util import run, run_output, root_path, build_path, executable_suffix import tempfile import http_server +import throughput_benchmark # The list of the tuples of the benchmark name and arguments exec_time_benchmarks = [ @@ -116,6 +117,15 @@ def run_thread_count_benchmark(deno_path): return thread_count_map +def run_throughput(deno_exe): + m = {} + m["100M_tcp"] = throughput_benchmark.tcp(deno_exe, 100) + m["100M_cat"] = throughput_benchmark.cat(deno_exe, 100) + m["10M_tcp"] = throughput_benchmark.tcp(deno_exe, 10) + m["10M_cat"] = throughput_benchmark.cat(deno_exe, 10) + return m + + def run_syscall_count_benchmark(deno_path): syscall_count_map = {} syscall_count_map["hello"] = get_strace_summary( @@ -169,6 +179,10 @@ def main(argv): } new_data["binary_size"] = get_binary_sizes(build_dir) + # Cannot run throughput benchmark on windows because they don't have nc or + # pipe. + if os.name != 'nt': + new_data["throughput"] = run_throughput(deno_path) if "linux" in sys.platform: # Thread count test, only on linux new_data["thread_count"] = run_thread_count_benchmark(deno_path) diff --git a/tools/format.py b/tools/format.py index 272f2c82b31fd6..5b6afd1c3643a9 100755 --- a/tools/format.py +++ b/tools/format.py @@ -34,6 +34,7 @@ find_exts(".github/", ".md") + find_exts("js/", ".js", ".ts", ".md") + find_exts("tests/", ".js", ".ts", ".md") + + find_exts("tools/", ".js", ".json", ".ts", ".md") + find_exts("website/", ".js", ".ts", ".md")) # yapf: enable diff --git a/tools/http_server.py b/tools/http_server.py index d33f24d5d49732..c627dfd5f6c1b0 100755 --- a/tools/http_server.py +++ b/tools/http_server.py @@ -55,5 +55,4 @@ def spawn(): if __name__ == '__main__': - s = server() - s.serve_forever() + spawn().join() diff --git a/tools/testdata/unit_test_output1.txt b/tools/testdata/unit_test_output1.txt new file mode 100644 index 00000000000000..4b208319f80816 --- /dev/null +++ b/tools/testdata/unit_test_output1.txt @@ -0,0 +1,238 @@ +running 96 tests +test permSerialization_permW0N0E0 +... ok +test permFromStringThrows_permW0N0E0 +... ok +test compilerInstance_permW0N0E0 +... ok +test compilerRun_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRunMultiModule_permW0N0E0 +... ok +test compilerRunCircularDependency_permW0N0E0 +Compiling modA +Compiling modB +... ok +test compilerResolveModule_permW0N0E0 +... ok +test compilerGetModuleDependencies_permW0N0E0 +... ok +test compilerGetCompilationSettings_permW0N0E0 +... ok +test compilerGetNewLine_permW0N0E0 +... ok +test compilerGetScriptFileNames_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRecompileFlag_permW0N0E0 +Compiling /root/project/foo/bar.ts +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptKind_permW0N0E0 +... ok +test compilerGetScriptVersion_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptVersionUnknown_permW0N0E0 +... ok +test compilerGetScriptSnapshot_permW0N0E0 +... ok +test compilerGetCurrentDirectory_permW0N0E0 +... ok +test compilerGetDefaultLibFileName_permW0N0E0 +... ok +test compilerUseCaseSensitiveFileNames_permW0N0E0 +... ok +test compilerReadFile_permW0N0E0 +... ok +test compilerFileExists_permW0N0E0 +... ok +test compilerResolveModuleNames_permW0N0E0 +... ok +test consoleTestAssert_permW0N0E0 +... ok +test consoleTestStringifyComplexObjects_permW0N0E0 +... ok +test consoleTestStringifyCircular_permW0N0E0 +... ok +test consoleTestStringifyWithDepth_permW0N0E0 +... ok +test consoleTestError_permW0N0E0 +... ok +test consoleDetachedLog_permW0N0E0 +Hello world +Hello world +Hello world +Hello world +Hello world +Hello world +... ok +test fetchPerm_permW0N0E0 +... ok +test headersAppend_permW0N0E0 +... ok +test newHeaderTest_permW0N0E0 +... ok +test newHeaderWithSequence_permW0N0E0 +... ok +test newHeaderWithRecord_permW0N0E0 +... ok +test newHeaderWithHeadersInstance_permW0N0E0 +... ok +test headerAppendSuccess_permW0N0E0 +... ok +test headerSetSuccess_permW0N0E0 +... ok +test headerHasSuccess_permW0N0E0 +... ok +test headerDeleteSuccess_permW0N0E0 +... ok +test headerGetSuccess_permW0N0E0 +... ok +test headerForEachSuccess_permW0N0E0 +... ok +test envFailure_permW0N0E0 +... ok +test filesStdioFileDescriptors_permW0N0E0 +... ok +test filesCopyToStdout_permW0N0E0 +{ + "name": "deno", + "devDependencies": { + "@types/base64-js": "^1.2.5", + "@types/flatbuffers": "^1.9.0", + "@types/source-map-support": "^0.4.1", + "@types/text-encoding": "0.0.33", + "base64-js": "^1.3.0", + "flatbuffers": "^1.9.0", + "magic-string": "^0.22.5", + "prettier": "^1.14.0", + "rollup": "^0.63.2", + "rollup-plugin-alias": "^1.4.0", + "rollup-plugin-analyzer": "^2.1.0", + "rollup-plugin-commonjs": "^9.1.3", + "rollup-plugin-node-globals": "^1.2.1", + "rollup-plugin-node-resolve": "^3.3.0", + "rollup-plugin-string": "^2.0.2", + "rollup-plugin-typescript2": "^0.16.1", + "rollup-pluginutils": "^2.3.0", + "source-map-support": "^0.5.6", + "text-encoding": "0.6.4", + "tslint": "^5.10.0", + "tslint-eslint-rules": "^5.3.1", + "tslint-no-circular-imports": "^0.5.0", + "typescript": "3.0.3" + } +} +bytes written 860 +... ok +test readFileSyncSuccess_permW0N0E0 +... ok +test readFileSyncNotFound_permW0N0E0 +... ok +test readFileSuccess_permW0N0E0 +... ok +test readDirSyncNotDir_permW0N0E0 +... ok +test readDirSyncNotFound_permW0N0E0 +... ok +test writeFileSyncPerm_permW0N0E0 +... ok +test writeFilePerm_permW0N0E0 +... ok +test copyFileSyncPerm_permW0N0E0 +... ok +test copyFilePerm_permW0N0E0 +... ok +test mkdirSyncPerm_permW0N0E0 +... ok +test makeTempDirSyncPerm_permW0N0E0 +... ok +test statSyncSuccess_permW0N0E0 +... ok +test statSyncNotFound_permW0N0E0 +... ok +test lstatSyncSuccess_permW0N0E0 +... ok +test lstatSyncNotFound_permW0N0E0 +... ok +test statSuccess_permW0N0E0 +... ok +test statNotFound_permW0N0E0 +... ok +test lstatSuccess_permW0N0E0 +... ok +test lstatNotFound_permW0N0E0 +... ok +test renameSyncPerm_permW0N0E0 +... ok +test readlinkSyncNotFound_permW0N0E0 +... ok +test blobString_permW0N0E0 +... ok +test blobBuffer_permW0N0E0 +... ok +test blobSlice_permW0N0E0 +... ok +test timeoutSuccess_permW0N0E0 +... ok +test timeoutArgs_permW0N0E0 +... ok +test timeoutCancelSuccess_permW0N0E0 +... ok +test timeoutCancelMultiple_permW0N0E0 +... ok +test timeoutCancelInvalidSilentFail_permW0N0E0 +... ok +test intervalSuccess_permW0N0E0 +... ok +test intervalCancelSuccess_permW0N0E0 +... ok +test intervalOrdering_permW0N0E0 +... ok +test intervalCancelInvalidSilentFail_permW0N0E0 +... ok +test symlinkSyncPerm_permW0N0E0 +... ok +test platformTransform_permW0N0E0 +... ok +test atobSuccess_permW0N0E0 +... ok +test btoaSuccess_permW0N0E0 +... ok +test btoaFailed_permW0N0E0 +... ok +test truncateSyncPerm_permW0N0E0 +... ok +test truncatePerm_permW0N0E0 +... ok +test evalErrorFormatted_permW0N0E0 +... ok +test createExecTimeColumnsRegularData_permW0N0E0 +... ok +test createExecTimeColumnsIrregularData_permW0N0E0 +... ok +test createBinarySizeColumnsRegularData_permW0N0E0 +... ok +test createBinarySizeColumnsIrregularData_permW0N0E0 +... ok +test createThreadCountColumnsRegularData_permW0N0E0 +... ok +test createThreadCountColumnsIrregularData_permW0N0E0 +... ok +test createSyscallCountColumnsRegularData_permW0N0E0 +... ok +test createSyscallCountColumnsIrregularData_permW0N0E0 +... ok +test createSha1ListRegularData_permW0N0E0 +... ok +test formatBytesPatterns_permW0N0E0 +... ok +test formatSecondsPatterns_permW0N0E0 +... ok +test getTravisDataSuccess_permW0N0E0 +... ok + +test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out + diff --git a/tools/testdata/unit_test_output2.txt b/tools/testdata/unit_test_output2.txt new file mode 100644 index 00000000000000..5913d3b90ebc26 --- /dev/null +++ b/tools/testdata/unit_test_output2.txt @@ -0,0 +1,71 @@ +running 96 tests +test permSerialization_permW0N0E0 +... ok +test permFromStringThrows_permW0N0E0 +... ok +test compilerInstance_permW0N0E0 +... ok +test compilerRun_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRunMultiModule_permW0N0E0 +... ok +test compilerRunCircularDependency_permW0N0E0 +Compiling modA +Compiling modB +... ok +test compilerResolveModule_permW0N0E0 +... ok +test compilerGetModuleDependencies_permW0N0E0 +... ok +test compilerGetCompilationSettings_permW0N0E0 +... ok +test compilerGetNewLine_permW0N0E0 +... ok +test compilerGetScriptFileNames_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRecompileFlag_permW0N0E0 +Compiling /root/project/foo/bar.ts +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptKind_permW0N0E0 +... ok +test compilerGetScriptVersion_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptVersionUnknown_permW0N0E0 +... ok +test compilerGetScriptSnapshot_permW0N0E0 +... ok +test compilerGetCurrentDirectory_permW0N0E0 +... ok +test compilerGetDefaultLibFileName_permW0N0E0 +... ok +test compilerUseCaseSensitiveFileNames_permW0N0E0 +... ok +test compilerReadFile_permW0N0E0 +... ok +test compilerFileExists_permW0N0E0 +... ok +test compilerResolveModuleNames_permW0N0E0 +... ok +test consoleTestAssert_permW0N0E0 +... ok +test consoleTestStringifyComplexObjects_permW0N0E0 +... ok +test consoleTestStringifyCircular_permW0N0E0 +... ok +test consoleTestStringifyWithDepth_permW0N0E0 +... ok +test consoleTestError_permW0N0E0 +... ok +test consoleDetachedLog_permW0N0E0 +Hello world +Hello world +Hello world +Hello world +Hello world +Hello world +... ok +test fetchPerm_permW0N0E0 diff --git a/tools/testdata/unit_test_output3.txt b/tools/testdata/unit_test_output3.txt new file mode 100644 index 00000000000000..402261e7682ee6 --- /dev/null +++ b/tools/testdata/unit_test_output3.txt @@ -0,0 +1,268 @@ +Compiling /Users/rld/src/deno/js/unit_tests.ts +Compiling /Users/rld/src/deno/js/compiler_test.ts +Compiling /Users/rld/src/deno/js/test_util.ts +Compiling /Users/rld/src/deno/js/testing/testing.ts +Compiling /Users/rld/src/deno/js/testing/util.ts +Compiling /Users/rld/src/deno/js/console_test.ts +Compiling /Users/rld/src/deno/js/console.ts +Compiling /Users/rld/src/deno/js/fetch_test.ts +Compiling /Users/rld/src/deno/js/os_test.ts +Compiling /Users/rld/src/deno/js/files_test.ts +Compiling /Users/rld/src/deno/js/read_file_test.ts +Compiling /Users/rld/src/deno/js/read_dir_test.ts +Compiling /Users/rld/src/deno/js/write_file_test.ts +Compiling /Users/rld/src/deno/js/copy_file_test.ts +Compiling /Users/rld/src/deno/js/mkdir_test.ts +Compiling /Users/rld/src/deno/js/make_temp_dir_test.ts +Compiling /Users/rld/src/deno/js/stat_test.ts +Compiling /Users/rld/src/deno/js/rename_test.ts +Compiling /Users/rld/src/deno/js/read_link_test.ts +Compiling /Users/rld/src/deno/js/blob_test.ts +Compiling /Users/rld/src/deno/js/timers_test.ts +Compiling /Users/rld/src/deno/js/symlink_test.ts +Compiling /Users/rld/src/deno/js/platform_test.ts +Compiling /Users/rld/src/deno/js/text_encoding_test.ts +Compiling /Users/rld/src/deno/js/net_test.ts +Compiling /Users/rld/src/deno/js/trace_test.ts +Compiling /Users/rld/src/deno/js/truncate_test.ts +Compiling /Users/rld/src/deno/js/v8_source_maps_test.ts +Compiling /Users/rld/src/deno/website/app_test.js +Compiling /Users/rld/src/deno/website/app.js +running 96 tests +test permSerialization_permW0N0E0 +... ok +test permFromStringThrows_permW0N0E0 +... ok +test compilerInstance_permW0N0E0 +... ok +test compilerRun_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRunMultiModule_permW0N0E0 +... ok +test compilerRunCircularDependency_permW0N0E0 +Compiling modA +Compiling modB +... ok +test compilerResolveModule_permW0N0E0 +... ok +test compilerGetModuleDependencies_permW0N0E0 +... ok +test compilerGetCompilationSettings_permW0N0E0 +... ok +test compilerGetNewLine_permW0N0E0 +... ok +test compilerGetScriptFileNames_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerRecompileFlag_permW0N0E0 +Compiling /root/project/foo/bar.ts +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptKind_permW0N0E0 +... ok +test compilerGetScriptVersion_permW0N0E0 +Compiling /root/project/foo/bar.ts +... ok +test compilerGetScriptVersionUnknown_permW0N0E0 +... ok +test compilerGetScriptSnapshot_permW0N0E0 +... ok +test compilerGetCurrentDirectory_permW0N0E0 +... ok +test compilerGetDefaultLibFileName_permW0N0E0 +... ok +test compilerUseCaseSensitiveFileNames_permW0N0E0 +... ok +test compilerReadFile_permW0N0E0 +... ok +test compilerFileExists_permW0N0E0 +... ok +test compilerResolveModuleNames_permW0N0E0 +... ok +test consoleTestAssert_permW0N0E0 +... ok +test consoleTestStringifyComplexObjects_permW0N0E0 +... ok +test consoleTestStringifyCircular_permW0N0E0 +... ok +test consoleTestStringifyWithDepth_permW0N0E0 +... ok +test consoleTestError_permW0N0E0 +... ok +test consoleDetachedLog_permW0N0E0 +Hello world +Hello world +Hello world +Hello world +Hello world +Hello world +... ok +test fetchPerm_permW0N0E0 +... ok +test headersAppend_permW0N0E0 +... ok +test newHeaderTest_permW0N0E0 +... ok +test newHeaderWithSequence_permW0N0E0 +... ok +test newHeaderWithRecord_permW0N0E0 +... ok +test newHeaderWithHeadersInstance_permW0N0E0 +... ok +test headerAppendSuccess_permW0N0E0 +... ok +test headerSetSuccess_permW0N0E0 +... ok +test headerHasSuccess_permW0N0E0 +... ok +test headerDeleteSuccess_permW0N0E0 +... ok +test headerGetSuccess_permW0N0E0 +... ok +test headerForEachSuccess_permW0N0E0 +... ok +test envFailure_permW0N0E0 +... ok +test filesStdioFileDescriptors_permW0N0E0 +... ok +test filesCopyToStdout_permW0N0E0 +{ + "name": "deno", + "devDependencies": { + "@types/base64-js": "^1.2.5", + "@types/flatbuffers": "^1.9.0", + "@types/source-map-support": "^0.4.1", + "@types/text-encoding": "0.0.33", + "base64-js": "^1.3.0", + "flatbuffers": "^1.9.0", + "magic-string": "^0.22.5", + "prettier": "^1.14.0", + "rollup": "^0.63.2", + "rollup-plugin-alias": "^1.4.0", + "rollup-plugin-analyzer": "^2.1.0", + "rollup-plugin-commonjs": "^9.1.3", + "rollup-plugin-node-globals": "^1.2.1", + "rollup-plugin-node-resolve": "^3.3.0", + "rollup-plugin-string": "^2.0.2", + "rollup-plugin-typescript2": "^0.16.1", + "rollup-pluginutils": "^2.3.0", + "source-map-support": "^0.5.6", + "text-encoding": "0.6.4", + "tslint": "^5.10.0", + "tslint-eslint-rules": "^5.3.1", + "tslint-no-circular-imports": "^0.5.0", + "typescript": "3.0.3" + } +} +bytes written 860 +... ok +test readFileSyncSuccess_permW0N0E0 +... ok +test readFileSyncNotFound_permW0N0E0 +... ok +test readFileSuccess_permW0N0E0 +... ok +test readDirSyncNotDir_permW0N0E0 +... ok +test readDirSyncNotFound_permW0N0E0 +... ok +test writeFileSyncPerm_permW0N0E0 +... ok +test writeFilePerm_permW0N0E0 +... ok +test copyFileSyncPerm_permW0N0E0 +... ok +test copyFilePerm_permW0N0E0 +... ok +test mkdirSyncPerm_permW0N0E0 +... ok +test makeTempDirSyncPerm_permW0N0E0 +... ok +test statSyncSuccess_permW0N0E0 +... ok +test statSyncNotFound_permW0N0E0 +... ok +test lstatSyncSuccess_permW0N0E0 +... ok +test lstatSyncNotFound_permW0N0E0 +... ok +test statSuccess_permW0N0E0 +... ok +test statNotFound_permW0N0E0 +... ok +test lstatSuccess_permW0N0E0 +... ok +test lstatNotFound_permW0N0E0 +... ok +test renameSyncPerm_permW0N0E0 +... ok +test readlinkSyncNotFound_permW0N0E0 +... ok +test blobString_permW0N0E0 +... ok +test blobBuffer_permW0N0E0 +... ok +test blobSlice_permW0N0E0 +... ok +test timeoutSuccess_permW0N0E0 +... ok +test timeoutArgs_permW0N0E0 +... ok +test timeoutCancelSuccess_permW0N0E0 +... ok +test timeoutCancelMultiple_permW0N0E0 +... ok +test timeoutCancelInvalidSilentFail_permW0N0E0 +... ok +test intervalSuccess_permW0N0E0 +... ok +test intervalCancelSuccess_permW0N0E0 +... ok +test intervalOrdering_permW0N0E0 +... ok +test intervalCancelInvalidSilentFail_permW0N0E0 +... ok +test symlinkSyncPerm_permW0N0E0 +... ok +test platformTransform_permW0N0E0 +... ok +test atobSuccess_permW0N0E0 +... ok +test btoaSuccess_permW0N0E0 +... ok +test btoaFailed_permW0N0E0 +... ok +test truncateSyncPerm_permW0N0E0 +... ok +test truncatePerm_permW0N0E0 +... ok +test evalErrorFormatted_permW0N0E0 +... ok +test createExecTimeColumnsRegularData_permW0N0E0 +... ok +test createExecTimeColumnsIrregularData_permW0N0E0 +... ok +test createBinarySizeColumnsRegularData_permW0N0E0 +... ok +test createBinarySizeColumnsIrregularData_permW0N0E0 +... ok +test createThreadCountColumnsRegularData_permW0N0E0 +... ok +test createThreadCountColumnsIrregularData_permW0N0E0 +... ok +test createSyscallCountColumnsRegularData_permW0N0E0 +... ok +test createSyscallCountColumnsIrregularData_permW0N0E0 +... ok +test createSha1ListRegularData_permW0N0E0 +... ok +test formatBytesPatterns_permW0N0E0 +... ok +test formatSecondsPatterns_permW0N0E0 +... ok +test getTravisDataSuccess_permW0N0E0 +... ok + +test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out + diff --git a/tools/throughput_benchmark.py b/tools/throughput_benchmark.py new file mode 100755 index 00000000000000..be3278b57f8652 --- /dev/null +++ b/tools/throughput_benchmark.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# Copyright 2018 the Deno authors. All rights reserved. MIT license. +# Performs benchmark and append data to //website/data.json. +# If //website/data.json doesn't exist, this script tries to import it from gh-pages branch. +# To view the results locally run ./tools/http_server.py and visit +# http://localhost:4545/website + +import os +import sys +import util +import time +import subprocess + +MB = 1024 * 1024 +ADDR = "127.0.0.1:4544" + + +def cat(deno_exe, megs): + size = megs * MB + start = time.time() + cmd = deno_exe + " tests/cat.ts /dev/zero | head -c %s " % size + print cmd + subprocess.check_output(cmd, shell=True) + end = time.time() + return end - start + + +def tcp(deno_exe, megs): + size = megs * MB + # Run deno echo server in the background. + echo_server = subprocess.Popen( + [deno_exe, "--allow-net", "tests/echo_server.ts", ADDR]) + + time.sleep(1) # wait for deno to wake up. TODO racy. + try: + start = time.time() + cmd = ("head -c %s /dev/zero " % size) + "| nc " + ADDR.replace( + ":", " ") + print cmd + subprocess.check_output(cmd, shell=True) + end = time.time() + return end - start + finally: + echo_server.kill() + + +if __name__ == '__main__': + deno_exe = sys.argv[1] + megs = int(sys.argv[2]) + if not deno_exe or not megs: + print "Usage ./tools/throughput_benchmark.py out/debug/deno 100" + sys.exit(1) + secs = tcp_throughput_benchmark(sys.argv[1], megs) + print secs, "seconds" diff --git a/tools/ts_library_builder/README.md b/tools/ts_library_builder/README.md new file mode 100644 index 00000000000000..de1305dca9ce25 --- /dev/null +++ b/tools/ts_library_builder/README.md @@ -0,0 +1,95 @@ +# ts_library_builder + +This tool allows us to produce a single TypeScript declaration file that +describes the complete Deno runtime, including global variables and the built-in +`deno` module. The output of this tool, `lib.deno_runtime.d.ts`, serves several +purposes: + +1. It is passed to the TypeScript compiler `js/compiler.ts`, so that TypeScript + knows what types to expect and can validate code against the runtime + environment. +2. It is outputted to stdout by `deno --types`, so that users can easily have + access to the complete declaration file. Editors can use this in the future + to perform type checking. +3. Because JSDocs are maintained, this serves as a simple documentation page for + Deno. We will use this file to generate HTML docs in the future. + +The tool depends upon a couple libraries: + +- [`ts-node`](https://www.npmjs.com/package/ts-node) to provide just in time + transpiling of TypeScript for the tool itself. +- [`ts-simple-ast`](https://www.npmjs.com/package/ts-simple-ast) which provides + a more rational and functional interface to the TypeScript AST to make + manipulations easier. +- [`prettier`](https://www.npmjs.com/package/prettier) and + [`@types/prettier`](https://www.npmjs.com/package/@types/prettier) to format + the output. + +## Design + +Ideally we wouldn't have to build this tool at all, and could simply use `tsc` +to output this declaration file. While, `--emitDeclarationsOnly`, `--outFile` +and `--module AMD` generates a single declaration file, it isn't clean. It was +never designed for a library generation, where what is available in a runtime +environment significantly differs from the code that creates that environment's +structure. + +Therefore this tool injects some of the knowledge of what occurs in the Deno +runtime environment as well as ensures that the output file is more clean and +logical for an end user. In the deno runtime, code runs in a global scope that +is defined in `js/global.ts`. This contains global scope items that one +reasonably expects in a JavaScript runtime, like `console`. It also defines the +global scope on a self-reflective `window` variable. There is currently only one +module of Deno specific APIs which is available to the user. This is defined in +`js/deno.ts`. + +This tool takes advantage of an experimental feature of TypeScript that items +that are not really intended to be part of the public API are marked with a +comment pragma of `@internal` and then are not emitted when generating type +definitions. In addition TypeScript will _tree-shake_ any dependencies tied to +that "hidden" API and elide them as well. This really helps keep the public API +clean and as minimal as needed. + +In order to create the default type library, the process at a high-level looks +like this: + +- We read in all of the runtime environment definition code into TypeScript AST + parser "project". +- We emit the TypeScript type definitions only into another AST parser + "project". +- We process the `deno` namespace/module, by "flattening" the type definition + file. + - We determine the exported symbols for `js/deno.ts`. + - We create a custom extraction of the `gen/msg_generated.ts` which is + generated during the build process and contains the type information related + to flatbuffer structures that communicate between the privileged part of + deno and the user land. Currently, the tool doesn't do full complex + dependency analysis to be able to determine what is required out of this + file, so we explicitly extract the type information we need. + - We recurse over all imports/exports of the modules, only exporting those + symbols which are finally exported by `js/deno.ts`. + - We replace the import/export with the type information from the source file. + - This process assumes that all the modules that feed `js/deno.ts` will have a + public type API that does not have name conflicts. +- We process the `js/globals.ts` file to generate the global namespace. + - Currently we create a `"globals"` module which will contain the type + definitions. + - We create a `Window` interface and a `global` scope augmentation namespace. + - We iterate over augmentations to the `window` variable declared in the file, + extract the type information and apply it to both a global variable + declaration and a property on the `Window` interface. +- We take each namespace import to `js/globals.ts`, we resolve the emitted + declaration `.d.ts` file and create it as its own namespace withing the + `"globals"` module. It is unsafe to just flatten these, because there is a + high risk of collisions, but also, it makes authoring the types easier within + the generated interface and variable declarations. +- We then validate the resulting definition file and write it out to the + appropriate build path. + +## TODO + +- The tool does not _tree-shake_ when flattening imports. This means there are + extraneous types that get included that are not really needed and it means + that `gen/msg_generated.ts` has to be explicitly carved down. +- Complete the tests... we have some coverage, but not a lot of what is in + `ast_util_test` which is being tested implicitly. diff --git a/tools/ts_library_builder/ast_util.ts b/tools/ts_library_builder/ast_util.ts new file mode 100644 index 00000000000000..c13195b08b4300 --- /dev/null +++ b/tools/ts_library_builder/ast_util.ts @@ -0,0 +1,331 @@ +import { relative } from "path"; +import { readFileSync } from "fs"; +import { EOL } from "os"; +import { + ExportDeclaration, + ImportDeclaration, + InterfaceDeclaration, + JSDoc, + Project, + PropertySignature, + SourceFile, + StatementedNode, + ts, + TypeGuards, + VariableStatement, + VariableDeclarationKind +} from "ts-simple-ast"; + +/** Add a property to an interface */ +export function addInterfaceProperty( + interfaceDeclaration: InterfaceDeclaration, + name: string, + type: string, + jsdocs?: JSDoc[] +): PropertySignature { + return interfaceDeclaration.addProperty({ + name, + type, + docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText()) + }); +} + +/** Add `@url` comment to node. */ +export function addSourceComment( + node: StatementedNode, + sourceFile: SourceFile, + rootPath: string +): void { + node.insertStatements( + 0, + `// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n` + ); +} + +/** Add a declaration of a variable to a node */ +export function addVariableDeclaration( + node: StatementedNode, + name: string, + type: string, + jsdocs?: JSDoc[] +): VariableStatement { + return node.addVariableStatement({ + declarationKind: VariableDeclarationKind.Const, + declarations: [{ name, type }], + docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText()) + }); +} + +/** Check diagnostics, and if any exist, exit the process */ +export function checkDiagnostics(project: Project, onlyFor?: string[]) { + const program = project.getProgram(); + const diagnostics = [ + ...program.getGlobalDiagnostics(), + ...program.getSyntacticDiagnostics(), + ...program.getSemanticDiagnostics(), + ...program.getDeclarationDiagnostics() + ] + .filter(diagnostic => { + const sourceFile = diagnostic.getSourceFile(); + return onlyFor && sourceFile + ? onlyFor.includes(sourceFile.getFilePath()) + : true; + }) + .map(diagnostic => diagnostic.compilerObject); + + if (diagnostics.length) { + console.log( + ts.formatDiagnosticsWithColorAndContext(diagnostics, formatDiagnosticHost) + ); + process.exit(1); + } +} + +export interface FlattenNamespaceOptions { + customSources?: { [sourceFilePath: string]: string }; + debug?: boolean; + rootPath: string; + sourceFile: SourceFile; +} + +/** Take a namespace and flatten all exports. */ +export function flattenNamespace({ + customSources, + debug, + rootPath, + sourceFile +}: FlattenNamespaceOptions): string { + const sourceFiles = new Set(); + let output = ""; + const exportedSymbols = getExportedSymbols(sourceFile); + + function flattenDeclarations( + declaration: ImportDeclaration | ExportDeclaration + ) { + const declarationSourceFile = declaration.getModuleSpecifierSourceFile(); + if (declarationSourceFile) { + processSourceFile(declarationSourceFile); + declaration.remove(); + } + } + + function rectifyNodes(currentSourceFile: SourceFile) { + currentSourceFile.forEachChild(node => { + if (TypeGuards.isAmbientableNode(node)) { + node.setHasDeclareKeyword(false); + } + if (TypeGuards.isExportableNode(node)) { + const nodeSymbol = node.getSymbol(); + if ( + nodeSymbol && + !exportedSymbols.has(nodeSymbol.getFullyQualifiedName()) + ) { + node.setIsExported(false); + } + } + }); + } + + function processSourceFile(currentSourceFile: SourceFile) { + if (sourceFiles.has(currentSourceFile)) { + return; + } + sourceFiles.add(currentSourceFile); + + const currentSourceFilePath = currentSourceFile.getFilePath(); + if (customSources && currentSourceFilePath in customSources) { + output += customSources[currentSourceFilePath]; + return; + } + + currentSourceFile.getImportDeclarations().forEach(flattenDeclarations); + currentSourceFile.getExportDeclarations().forEach(flattenDeclarations); + + rectifyNodes(currentSourceFile); + + output += + (debug ? getSourceComment(currentSourceFile, rootPath) : "") + + currentSourceFile.print(); + } + + sourceFile.getExportDeclarations().forEach(exportDeclaration => { + processSourceFile(exportDeclaration.getModuleSpecifierSourceFileOrThrow()); + exportDeclaration.remove(); + }); + + rectifyNodes(sourceFile); + + return ( + output + + (debug ? getSourceComment(sourceFile, rootPath) : "") + + sourceFile.print() + ); +} + +/** Used when formatting diagnostics */ +const formatDiagnosticHost: ts.FormatDiagnosticsHost = { + getCurrentDirectory() { + return process.cwd(); + }, + getCanonicalFileName(path: string) { + return path; + }, + getNewLine() { + return EOL; + } +}; + +/** Return a set of fully qualified symbol names for the files exports */ +function getExportedSymbols(sourceFile: SourceFile): Set { + const exportedSymbols = new Set(); + const exportDeclarations = sourceFile.getExportDeclarations(); + for (const exportDeclaration of exportDeclarations) { + const exportSpecifiers = exportDeclaration.getNamedExports(); + for (const exportSpecifier of exportSpecifiers) { + const aliasedSymbol = exportSpecifier + .getSymbolOrThrow() + .getAliasedSymbol(); + if (aliasedSymbol) { + exportedSymbols.add(aliasedSymbol.getFullyQualifiedName()); + } + } + } + return exportedSymbols; +} + +/** Returns a string which indicates the source file as the source */ +export function getSourceComment( + sourceFile: SourceFile, + rootPath: string +): string { + return `\n// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`; +} + +/** + * Load and write to a virtual file system all the default libs needed to + * resolve types on project. + */ +export function loadDtsFiles(project: Project) { + loadFiles( + project, + [ + "lib.es2015.collection.d.ts", + "lib.es2015.core.d.ts", + "lib.es2015.d.ts", + "lib.es2015.generator.d.ts", + "lib.es2015.iterable.d.ts", + "lib.es2015.promise.d.ts", + "lib.es2015.proxy.d.ts", + "lib.es2015.reflect.d.ts", + "lib.es2015.symbol.d.ts", + "lib.es2015.symbol.wellknown.d.ts", + "lib.es2016.array.include.d.ts", + "lib.es2016.d.ts", + "lib.es2017.d.ts", + "lib.es2017.intl.d.ts", + "lib.es2017.object.d.ts", + "lib.es2017.sharedmemory.d.ts", + "lib.es2017.string.d.ts", + "lib.es2017.typedarrays.d.ts", + "lib.es2018.d.ts", + "lib.es2018.intl.d.ts", + "lib.es2018.promise.d.ts", + "lib.es2018.regexp.d.ts", + "lib.es5.d.ts", + "lib.esnext.d.ts", + "lib.esnext.array.d.ts", + "lib.esnext.asynciterable.d.ts", + "lib.esnext.intl.d.ts", + "lib.esnext.symbol.d.ts" + ].map(fileName => `node_modules/typescript/lib/${fileName}`) + ); +} + +/** Load a set of files into a file system host. */ +export function loadFiles(project: Project, filePaths: string[]) { + const fileSystem = project.getFileSystem(); + for (const filePath of filePaths) { + const fileText = readFileSync(filePath, { + encoding: "utf8" + }); + fileSystem.writeFileSync(filePath, fileText); + } +} + +export interface NamespaceSourceFileOptions { + debug?: boolean; + namespace?: string; + rootPath: string; + sourceFileMap: Map; +} + +/** + * Take a source file (`.d.ts`) and convert it to a namespace, resolving any + * imports as their own namespaces. + */ +export function namespaceSourceFile( + sourceFile: SourceFile, + { debug, namespace, rootPath, sourceFileMap }: NamespaceSourceFileOptions +): string { + if (sourceFileMap.has(sourceFile)) { + return ""; + } + if (!namespace) { + namespace = sourceFile.getBaseNameWithoutExtension(); + } + sourceFileMap.set(sourceFile, namespace); + + sourceFile.forEachChild(node => { + if (TypeGuards.isAmbientableNode(node)) { + node.setHasDeclareKeyword(false); + } + }); + + const globalNamespace = sourceFile.getNamespace("global"); + const globalNamespaceText = globalNamespace && globalNamespace.print(); + if (globalNamespace) { + globalNamespace.remove(); + } + + const output = sourceFile + .getImportDeclarations() + .map(declaration => { + if ( + declaration.getNamedImports().length || + !declaration.getNamespaceImport() + ) { + throw new Error( + "Unsupported import clause.\n" + + ` In: "${declaration.getSourceFile().getFilePath()}"\n` + + ` Text: "${declaration.getText()}"` + ); + } + const text = namespaceSourceFile( + declaration.getModuleSpecifierSourceFileOrThrow(), + { + debug, + namespace: declaration.getNamespaceImportOrThrow().getText(), + rootPath, + sourceFileMap + } + ); + declaration.remove(); + return text; + }) + .join("\n"); + sourceFile + .getExportDeclarations() + .forEach(declaration => declaration.remove()); + + return `${output} + ${globalNamespaceText || ""} + namespace ${namespace} { + ${debug ? getSourceComment(sourceFile, rootPath) : ""} + ${sourceFile.getText()} + }`; +} + +/** Mirrors TypeScript's handling of paths */ +export function normalizeSlashes(path: string): string { + return path.replace(/\\/g, "/"); +} diff --git a/tools/ts_library_builder/build_library.ts b/tools/ts_library_builder/build_library.ts new file mode 100644 index 00000000000000..589e26a822977e --- /dev/null +++ b/tools/ts_library_builder/build_library.ts @@ -0,0 +1,453 @@ +import { writeFileSync } from "fs"; +import * as prettier from "prettier"; +import { + ExpressionStatement, + ModuleKind, + ModuleResolutionKind, + NamespaceDeclarationKind, + Project, + ScriptTarget, + SourceFile, + Type, + TypeGuards +} from "ts-simple-ast"; +import { + addInterfaceProperty, + addSourceComment, + addVariableDeclaration, + checkDiagnostics, + flattenNamespace, + getSourceComment, + loadDtsFiles, + loadFiles, + namespaceSourceFile, + normalizeSlashes +} from "./ast_util"; + +export interface BuildLibraryOptions { + /** + * The path to the root of the deno repository + */ + basePath: string; + + /** + * The path to the current build path + */ + buildPath: string; + + /** + * Denotes if the library should be built with debug information (comments + * that indicate the source of the types) + */ + debug?: boolean; + + /** + * The path to the output library + */ + outFile: string; + + /** + * Execute in silent mode or not + */ + silent?: boolean; +} + +/** + * A preamble which is appended to the start of the library. + */ +// tslint:disable-next-line:max-line-length +const libPreamble = `// Copyright 2018 the Deno authors. All rights reserved. MIT license. + +/// +/// + +`; + +// The path to the msg_generated file relative to the build path +const MSG_GENERATED_PATH = "/gen/msg_generated.ts"; + +// An array of enums we want to expose pub +const MSG_GENERATED_ENUMS = ["ErrorKind"]; + +/** Extracts enums from a source file */ +function extract(sourceFile: SourceFile, enumNames: string[]): string { + // Copy specified enums from msg_generated + let output = ""; + for (const enumName of enumNames) { + const enumDeclaration = sourceFile.getEnumOrThrow(enumName); + enumDeclaration.setHasDeclareKeyword(false); + // we are not copying JSDocs or other trivia here because msg_generated only + // contains some non-useful JSDocs and comments that are not ideal to copy + // over + output += enumDeclaration.getText(); + } + return output; +} + +interface FlattenOptions { + basePath: string; + customSources: { [filePath: string]: string }; + filePath: string; + debug?: boolean; + declarationProject: Project; + namespaceName: string; + targetSourceFile: SourceFile; +} + +/** Flatten a module */ +export function flatten({ + basePath, + customSources, + filePath, + debug, + declarationProject, + namespaceName, + targetSourceFile +}: FlattenOptions): void { + // Flatten the source file into a single module declaration + const statements = flattenNamespace({ + sourceFile: declarationProject.getSourceFileOrThrow(filePath), + rootPath: basePath, + customSources, + debug + }); + + // Create the module in the target file + const namespace = targetSourceFile.addNamespace({ + name: namespaceName, + hasDeclareKeyword: true, + declarationKind: NamespaceDeclarationKind.Module + }); + + // Add the output of the flattening to the namespace + namespace.addStatements(statements); +} + +interface MergeOptions { + basePath: string; + declarationProject: Project; + debug?: boolean; + globalVarName: string; + filePath: string; + inputProject: Project; + interfaceName: string; + namespaceName: string; + targetSourceFile: SourceFile; +} + +/** Take a module and merge into into a single namespace */ +export function merge({ + basePath, + declarationProject, + debug, + globalVarName, + filePath, + inputProject, + interfaceName, + namespaceName, + targetSourceFile +}: MergeOptions) { + // We have to build the module/namespace in small pieces which will reflect + // how the global runtime environment will be for Deno + + // We need to add a module named `"globals"` which will contain all the global + // runtime context + const mergedModule = targetSourceFile.addNamespace({ + name: namespaceName, + hasDeclareKeyword: true, + declarationKind: NamespaceDeclarationKind.Module + }); + + // Add the global Window interface + const interfaceDeclaration = mergedModule.addInterface({ + name: interfaceName + }); + + // Add the global scope augmentation module of the "globals" module + const mergedGlobalNamespace = mergedModule.addNamespace({ + name: "global", + declarationKind: NamespaceDeclarationKind.Global + }); + + // Declare the global variable + addVariableDeclaration(mergedGlobalNamespace, globalVarName, interfaceName); + + // Add self reference to the global variable + addInterfaceProperty(interfaceDeclaration, globalVarName, interfaceName); + + // Retrieve source file from the input project + const sourceFile = inputProject.getSourceFileOrThrow(filePath); + + // we are going to create a map of variables + const globalVariables = new Map< + string, + { + type: Type; + node: ExpressionStatement; + } + >(); + + // For every augmentation of the global variable in source file, we want + // to extract the type and add it to the global variable map + sourceFile.forEachChild(node => { + if (TypeGuards.isExpressionStatement(node)) { + const firstChild = node.getFirstChild(); + if (!firstChild) { + return; + } + if (TypeGuards.isBinaryExpression(firstChild)) { + const leftExpression = firstChild.getLeft(); + if ( + TypeGuards.isPropertyAccessExpression(leftExpression) && + leftExpression.getExpression().getText() === globalVarName + ) { + const windowProperty = leftExpression.getName(); + if (windowProperty !== globalVarName) { + globalVariables.set(windowProperty, { + type: firstChild.getType(), + node + }); + } + } + } + } + }); + + // A set of source files that the types we are using are dependent on us + // importing + const dependentSourceFiles = new Set(); + + // Create a global variable and add the property to the `Window` interface + // for each mutation of the `window` variable we observed in `globals.ts` + for (const [property, info] of globalVariables) { + const type = info.type.getText(info.node); + const typeSymbol = info.type.getSymbol(); + if (typeSymbol) { + const valueDeclaration = typeSymbol.getValueDeclaration(); + if (valueDeclaration) { + dependentSourceFiles.add(valueDeclaration.getSourceFile()); + } + } + addVariableDeclaration(mergedGlobalNamespace, property, type); + addInterfaceProperty(interfaceDeclaration, property, type); + } + + // We need to ensure that we only namespace each source file once, so we + // will use this map for tracking that. + const sourceFileMap = new Map(); + + // For each import declaration in source file we will want to convert the + // declaration source file into a namespace that exists within the merged + // namespace + const importDeclarations = sourceFile.getImportDeclarations(); + for (const declaration of importDeclarations) { + const declarationSourceFile = declaration.getModuleSpecifierSourceFile(); + if ( + declarationSourceFile && + dependentSourceFiles.has(declarationSourceFile) + ) { + // the source file will resolve to the original `.ts` file, but the + // information we really want is in the emitted `.d.ts` file, so we will + // resolve to that file + const dtsFilePath = declarationSourceFile + .getFilePath() + .replace(/\.ts$/, ".d.ts"); + const dtsSourceFile = declarationProject.getSourceFileOrThrow( + dtsFilePath + ); + mergedModule.addStatements( + namespaceSourceFile(dtsSourceFile, { + debug, + namespace: declaration.getNamespaceImportOrThrow().getText(), + rootPath: basePath, + sourceFileMap + }) + ); + } + } + + if (debug) { + addSourceComment(mergedModule, sourceFile, basePath); + } +} + +/** + * Generate the runtime library for Deno and write it to the supplied out file + * name. + */ +export function main({ + basePath, + buildPath, + debug, + outFile, + silent +}: BuildLibraryOptions) { + if (!silent) { + console.log("-----"); + console.log("build_lib"); + console.log(); + console.log(`basePath: "${basePath}"`); + console.log(`buildPath: "${buildPath}"`); + console.log(`debug: ${!!debug}`); + console.log(`outFile: "${outFile}"`); + console.log(); + } + + // the inputProject will take in the TypeScript files that are internal + // to Deno to be used to generate the library + const inputProject = new Project({ + compilerOptions: { + baseUrl: basePath, + declaration: true, + emitDeclarationOnly: true, + lib: [], + module: ModuleKind.AMD, + moduleResolution: ModuleResolutionKind.NodeJs, + noLib: true, + paths: { + "*": ["*", `${buildPath}/*`] + }, + preserveConstEnums: true, + strict: true, + stripInternal: true, + target: ScriptTarget.ESNext + } + }); + + // Add the input files we will need to generate the declarations, `globals` + // plus any modules that are importable in the runtime need to be added here + // plus the `lib.esnext` which is used as the base library + inputProject.addExistingSourceFiles([ + `${basePath}/node_modules/typescript/lib/lib.esnext.d.ts`, + `${basePath}/js/deno.ts`, + `${basePath}/js/globals.ts` + ]); + + // emit the project, which will be only the declaration files + const inputEmitResult = inputProject.emitToMemory(); + + // the declaration project will be the target for the emitted files from + // the input project, these will be used to transfer information over to + // the final library file + const declarationProject = new Project({ + compilerOptions: { + baseUrl: basePath, + moduleResolution: ModuleResolutionKind.NodeJs, + noLib: true, + paths: { + "*": ["*", `${buildPath}/*`] + }, + strict: true, + target: ScriptTarget.ESNext + }, + useVirtualFileSystem: true + }); + + // we don't want to add to the declaration project any of the original + // `.ts` source files, so we need to filter those out + const jsPath = normalizeSlashes(`${basePath}/js`); + const inputProjectFiles = inputProject + .getSourceFiles() + .map(sourceFile => sourceFile.getFilePath()) + .filter(filePath => !filePath.startsWith(jsPath)); + loadFiles(declarationProject, inputProjectFiles); + + // now we add the emitted declaration files from the input project + for (const { filePath, text } of inputEmitResult.getFiles()) { + declarationProject.createSourceFile(filePath, text); + } + + // the outputProject will contain the final library file we are looking to + // build + const outputProject = new Project({ + compilerOptions: { + baseUrl: buildPath, + moduleResolution: ModuleResolutionKind.NodeJs, + noLib: true, + strict: true, + target: ScriptTarget.ESNext, + types: ["text-encoding"] + }, + useVirtualFileSystem: true + }); + + // There are files we need to load into memory, so that the project "compiles" + loadDtsFiles(outputProject); + // tslint:disable-next-line:max-line-length + const textEncodingFilePath = `${buildPath}/node_modules/@types/text-encoding/index.d.ts`; + loadFiles(outputProject, [textEncodingFilePath]); + outputProject.addExistingSourceFileIfExists(textEncodingFilePath); + + // libDts is the final output file we are looking to build and we are not + // actually creating it, only in memory at this stage. + const libDTs = outputProject.createSourceFile(outFile); + + // Deal with `js/deno.ts` + + // `gen/msg_generated.d.ts` contains too much exported information that is not + // part of the public API surface of Deno, so we are going to extract just the + // information we need. + const msgGeneratedDts = inputProject.getSourceFileOrThrow( + `${buildPath}${MSG_GENERATED_PATH}` + ); + const msgGeneratedDtsText = extract(msgGeneratedDts, MSG_GENERATED_ENUMS); + + // Generate a object hash of substitutions of modules to use when flattening + const customSources = { + [msgGeneratedDts.getFilePath()]: `${ + debug ? getSourceComment(msgGeneratedDts, basePath) : "" + }${msgGeneratedDtsText}\n` + }; + + flatten({ + basePath, + customSources, + debug, + declarationProject, + filePath: `${basePath}/js/deno.d.ts`, + namespaceName: `"deno"`, + targetSourceFile: libDTs + }); + + if (!silent) { + console.log(`Created module "deno".`); + } + + merge({ + basePath, + declarationProject, + debug, + globalVarName: "window", + filePath: `${basePath}/js/globals.ts`, + inputProject, + interfaceName: "Window", + namespaceName: `"globals"`, + targetSourceFile: libDTs + }); + + if (!silent) { + console.log(`Created module "globals".`); + } + + // Add the preamble + libDTs.insertStatements(0, libPreamble); + + // Check diagnostics + checkDiagnostics(outputProject); + + // Output the final library file + libDTs.saveSync(); + const libDTsText = prettier.format( + outputProject.getFileSystem().readFileSync(outFile, "utf8"), + { parser: "typescript" } + ); + if (!silent) { + console.log(`Outputting library to: "${outFile}"`); + console.log(` Length: ${libDTsText.length}`); + } + writeFileSync(outFile, libDTsText, { encoding: "utf8" }); + if (!silent) { + console.log("-----"); + console.log(); + } +} diff --git a/tools/ts_library_builder/main.ts b/tools/ts_library_builder/main.ts new file mode 100644 index 00000000000000..e6662577f10426 --- /dev/null +++ b/tools/ts_library_builder/main.ts @@ -0,0 +1,39 @@ +import * as path from "path"; +import { main as buildRuntimeLib } from "./build_library"; + +// this is very simplistic argument parsing, just enough to integrate into +// the build scripts, versus being very robust +let basePath = process.cwd(); +let buildPath = path.join(basePath, "out", "debug"); +let outFile = path.join(buildPath, "gen", "lib", "lib.d.ts"); +let debug = false; +let silent = false; + +process.argv.forEach((arg, i, argv) => { + // tslint:disable-next-line:switch-default + switch (arg) { + case "--basePath": + basePath = path.resolve(argv[i + 1]); + break; + case "--buildPath": + buildPath = path.resolve(argv[i + 1]); + break; + case "--outFile": + outFile = path.resolve(argv[i + 1]); + break; + case "--debug": + debug = true; + break; + case "--silent": + silent = true; + break; + } +}); + +buildRuntimeLib({ + basePath, + buildPath, + debug, + outFile, + silent +}); diff --git a/tools/ts_library_builder/test.ts b/tools/ts_library_builder/test.ts new file mode 100644 index 00000000000000..e5317b3939d89f --- /dev/null +++ b/tools/ts_library_builder/test.ts @@ -0,0 +1,159 @@ +// Run this manually with: +// +// ./node_modules/.bin/ts-node --project tools/ts_library_builder/tsconfig.json tools/ts_library_builder/test.ts + +import { + ModuleKind, + ModuleResolutionKind, + Project, + ScriptTarget +} from "ts-simple-ast"; +import { assert, assertEqual, test } from "../../js/testing/testing"; +import { flatten, merge } from "./build_library"; +import { loadDtsFiles } from "./ast_util"; + +/** setups and returns the fixtures for testing */ +function setupFixtures() { + const basePath = process.cwd(); + const buildPath = `${basePath}/tools/ts_library_builder/testdata`; + const outputFile = `${buildPath}/lib.output.d.ts`; + const inputProject = new Project({ + compilerOptions: { + baseUrl: basePath, + declaration: true, + emitDeclarationOnly: true, + module: ModuleKind.AMD, + moduleResolution: ModuleResolutionKind.NodeJs, + strict: true, + stripInternal: true, + target: ScriptTarget.ESNext + } + }); + inputProject.addExistingSourceFiles([ + `${buildPath}/globals.ts`, + `${buildPath}/api.ts` + ]); + const declarationProject = new Project({ + compilerOptions: {}, + useVirtualFileSystem: true + }); + loadDtsFiles(declarationProject); + for (const { filePath, text } of inputProject.emitToMemory().getFiles()) { + declarationProject.createSourceFile(filePath, text); + } + const outputProject = new Project({ + compilerOptions: {}, + useVirtualFileSystem: true + }); + loadDtsFiles(outputProject); + const outputSourceFile = outputProject.createSourceFile(outputFile); + const debug = true; + + return { + basePath, + buildPath, + inputProject, + outputFile, + declarationProject, + outputProject, + outputSourceFile, + debug + }; +} + +test(function buildLibraryFlatten() { + const { + basePath, + buildPath, + debug, + declarationProject, + outputSourceFile: targetSourceFile + } = setupFixtures(); + + flatten({ + basePath, + customSources: {}, + debug, + declarationProject, + filePath: `${buildPath}/api.d.ts`, + namespaceName: `"api"`, + targetSourceFile + }); + + assert(targetSourceFile.getNamespace(`"api"`) != null); + assertEqual(targetSourceFile.getNamespaces().length, 1); + const namespaceApi = targetSourceFile.getNamespaceOrThrow(`"api"`); + const functions = namespaceApi.getFunctions(); + assertEqual(functions[0].getName(), "foo"); + assertEqual( + functions[0] + .getJsDocs() + .map(jsdoc => jsdoc.getInnerText()) + .join("\n"), + "jsdoc for foo" + ); + assertEqual(functions[1].getName(), "bar"); + assertEqual( + functions[1] + .getJsDocs() + .map(jsdoc => jsdoc.getInnerText()) + .join("\n"), + "" + ); + assertEqual(functions.length, 2); + const classes = namespaceApi.getClasses(); + assertEqual(classes[0].getName(), "Foo"); + assertEqual(classes.length, 1); + const variableDeclarations = namespaceApi.getVariableDeclarations(); + assertEqual(variableDeclarations[0].getName(), "arr"); + assertEqual(variableDeclarations.length, 1); +}); + +test(function buildLibraryMerge() { + const { + basePath, + buildPath, + declarationProject, + debug, + inputProject, + outputSourceFile: targetSourceFile + } = setupFixtures(); + + merge({ + basePath, + declarationProject, + debug, + globalVarName: "foobarbaz", + filePath: `${buildPath}/globals.ts`, + inputProject, + interfaceName: "FooBar", + namespaceName: `"bazqat"`, + targetSourceFile + }); + + assert(targetSourceFile.getNamespace(`"bazqat"`) != null); + assertEqual(targetSourceFile.getNamespaces().length, 1); + const namespaceBazqat = targetSourceFile.getNamespaceOrThrow(`"bazqat"`); + assert(namespaceBazqat.getNamespace("global") != null); + assert(namespaceBazqat.getNamespace("moduleC") != null); + assertEqual(namespaceBazqat.getNamespaces().length, 2); + assert(namespaceBazqat.getInterface("FooBar") != null); + assertEqual(namespaceBazqat.getInterfaces().length, 1); + const globalNamespace = namespaceBazqat.getNamespaceOrThrow("global"); + const variableDeclarations = globalNamespace.getVariableDeclarations(); + assertEqual( + variableDeclarations[0].getType().getText(), + `import("bazqat").FooBar` + ); + assertEqual( + variableDeclarations[1].getType().getText(), + `import("bazqat").moduleC.Bar` + ); + assertEqual( + variableDeclarations[2].getType().getText(), + `typeof import("bazqat").moduleC.qat` + ); + assertEqual(variableDeclarations.length, 3); +}); + +// TODO author unit tests for `ast_util.ts` diff --git a/tools/ts_library_builder/testdata/api.ts b/tools/ts_library_builder/testdata/api.ts new file mode 100644 index 00000000000000..f282a84141f5f7 --- /dev/null +++ b/tools/ts_library_builder/testdata/api.ts @@ -0,0 +1,4 @@ +export { foo, bar } from "./moduleA"; +export { Foo } from "./moduleB"; +/** jsdoc for arr */ +export const arr: string[] = []; diff --git a/tools/ts_library_builder/testdata/globals.ts b/tools/ts_library_builder/testdata/globals.ts new file mode 100644 index 00000000000000..41a86bdf8896c0 --- /dev/null +++ b/tools/ts_library_builder/testdata/globals.ts @@ -0,0 +1,6 @@ +import * as moduleC from "./moduleC"; + +// tslint:disable-next-line:no-any +const foobarbaz: any = {}; +foobarbaz.bar = new moduleC.Bar(); +foobarbaz.qat = moduleC.qat; diff --git a/tools/ts_library_builder/testdata/moduleA.ts b/tools/ts_library_builder/testdata/moduleA.ts new file mode 100644 index 00000000000000..a5bd07e2fcb435 --- /dev/null +++ b/tools/ts_library_builder/testdata/moduleA.ts @@ -0,0 +1,9 @@ +/** jsdoc for foo */ +export function foo(a: string, b: string) { + console.log(a, b); +} + +// no jsdoc for bar +export async function bar(promise: Promise): Promise { + return promise.then(() => {}); +} diff --git a/tools/ts_library_builder/testdata/moduleB.ts b/tools/ts_library_builder/testdata/moduleB.ts new file mode 100644 index 00000000000000..91f5ea875722fd --- /dev/null +++ b/tools/ts_library_builder/testdata/moduleB.ts @@ -0,0 +1,9 @@ +/** jsdoc about Foo */ +export class Foo { + private _foo = "foo"; + /** jsdoc about Foo.log() */ + log() { + console.log(this._foo); + return this._foo; + } +} diff --git a/tools/ts_library_builder/testdata/moduleC.ts b/tools/ts_library_builder/testdata/moduleC.ts new file mode 100644 index 00000000000000..b998c9e9dbaa06 --- /dev/null +++ b/tools/ts_library_builder/testdata/moduleC.ts @@ -0,0 +1,18 @@ +/** jsdoc for Bar */ +export class Bar { + private _bar: string; + /** jsdoc for Bar.log() */ + log() { + console.log(this._bar); + return this.log; + } +} + +/** + * jsdoc for qat + * @param a jsdoc for qat(a) + * @param b jsdoc for qat(b) + */ +export function qat(a: string, b: string) { + return a + b; +} diff --git a/tools/ts_library_builder/tsconfig.json b/tools/ts_library_builder/tsconfig.json new file mode 100644 index 00000000000000..6d010335c281f4 --- /dev/null +++ b/tools/ts_library_builder/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "moduleResolution": "node", + "strict": true, + "target": "esnext" + }, + "files": ["./build_library.ts"] +} diff --git a/tools/unit_tests.py b/tools/unit_tests.py index 62844fc09819dd..14f1d18b7644f6 100755 --- a/tools/unit_tests.py +++ b/tools/unit_tests.py @@ -1,6 +1,34 @@ #!/usr/bin/env python -from util import run +import util import sys +import subprocess +import re + +def run_unit_test2(cmd): + process = subprocess.Popen( + cmd, + bufsize=1, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + (actual, expected) = util.parse_unit_test_output(process.stdout, True) + process.wait() + errcode = process.returncode + if errcode != 0: + sys.exit(errcode) + if actual == None and expected == None: + raise AssertionError("Bad js/unit_test.ts output") + if expected != actual: + print "expected", expected, "actual", actual + raise AssertionError("expected tests did not equal actual") + process.wait() + errcode = process.returncode + if errcode != 0: + sys.exit(errcode) + +def run_unit_test(deno_exe, permStr, flags=[]): + cmd = [deno_exe, "--reload", "js/unit_tests.ts", permStr] + flags + run_unit_test2(cmd) # We want to test many ops in deno which have different behavior depending on @@ -10,24 +38,21 @@ # tests by the special string. permW0N0 means allow-write but not allow-net. # See js/test_util.ts for more details. def unit_tests(deno_exe): - run([deno_exe, "--reload", "js/unit_tests.ts", "permW0N0E0"]) - run([ - deno_exe, "--reload", "js/unit_tests.ts", "permW1N0E0", "--allow-write" - ]) - run([ - deno_exe, "--reload", "js/unit_tests.ts", "permW0N1E0", "--allow-net" - ]) - run([ - deno_exe, "--reload", "js/unit_tests.ts", "permW0N0E1", "--allow-env" - ]) - run([ - deno_exe, - "--reload", - "js/unit_tests.ts", - "permW1N1E1", - "--allow-write", - "--allow-net", - "--allow-env", + run_unit_test(deno_exe, "permW0N0E0") + run_unit_test(deno_exe, "permW1N0E0", ["--allow-write"]) + run_unit_test(deno_exe, "permW0N1E0", ["--allow-net"]) + run_unit_test(deno_exe, "permW0N0E1", ["--allow-env"]) + # TODO We might accidentally miss some. We should be smarter about which we + # run. Maybe we can use the "filtered out" number to check this. + + # These are not strictly unit tests for Deno, but for ts_library_builder. + # They run under Node, but use the same //js/testing/ library. + run_unit_test2([ + "node", + "./node_modules/.bin/ts-node", + "--project", + "tools/ts_library_builder/tsconfig.json", + "tools/ts_library_builder/test.ts" ]) diff --git a/tools/util.py b/tools/util.py index afe4690ce852c6..2620e706fc41db 100644 --- a/tools/util.py +++ b/tools/util.py @@ -294,3 +294,33 @@ def check(result, func, args): CloseHandle(conout) return True + + +def parse_unit_test_output(output, print_to_stdout): + first = True + expected = None + actual = None + result = None + for line in iter(output.readline, ''): + if expected is None: + # expect "running 30 tests" + expected = extract_number(r'running (\d+) tests', line) + elif "test result:" in line: + result = line + if print_to_stdout: + sys.stdout.write(line) + sys.stdout.flush() + # Check that the number of expected tests equals what was reported at the + # bottom. + if result: + # result should be a string like this: + # "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; ..." + actual = extract_number(r'(\d+) passed', result) + return (actual, expected) + + +def extract_number(pattern, string): + matches = re.findall(pattern, string) + if len(matches) != 1: + return None + return int(matches[0]) diff --git a/tools/util_test.py b/tools/util_test.py index e40d0aed0dabf8..4adf0d658bd536 100644 --- a/tools/util_test.py +++ b/tools/util_test.py @@ -1,5 +1,8 @@ # Copyright 2018 the Deno authors. All rights reserved. MIT license. from util import pattern_match, parse_exit_code, shell_quote_win +import util +import os +import sys def pattern_match_test(): @@ -44,10 +47,42 @@ def shell_quote_win_test(): 'a"b""c\\d\\"e\\\\') +def parse_unit_test_output_test(): + print "Testing util.parse_unit_test_output()..." + # This is an example of a successful unit test output. + output = open( + os.path.join(util.root_path, "tools/testdata/unit_test_output1.txt")) + (actual, expected) = util.parse_unit_test_output(output, False) + assert actual == 96 + assert expected == 96 + + # This is an example of a silently dying unit test. + output = open( + os.path.join(util.root_path, "tools/testdata/unit_test_output2.txt")) + (actual, expected) = util.parse_unit_test_output(output, False) + assert actual == None + assert expected == 96 + + # This is an example of compiling before successful unit tests. + output = open( + os.path.join(util.root_path, "tools/testdata/unit_test_output3.txt")) + (actual, expected) = util.parse_unit_test_output(output, False) + assert actual == 96 + assert expected == 96 + + # Check what happens on empty output. + from StringIO import StringIO + output = StringIO("\n\n\n") + (actual, expected) = util.parse_unit_test_output(output, False) + assert actual == None + assert expected == None + + def util_test(): pattern_match_test() parse_exit_code_test() shell_quote_win_test() + parse_unit_test_output_test() if __name__ == '__main__': diff --git a/website/app.js b/website/app.js index 403aa343a2cd44..7346defb4de681 100644 --- a/website/app.js +++ b/website/app.js @@ -16,18 +16,41 @@ export function getTravisData() { .then(data => data.builds.reverse()); } -export function createExecTimeColumns(data) { - const benchmarkNames = Object.keys(data[data.length - 1].benchmark); - return benchmarkNames.map(name => [ - name, +function getBenchmarkVarieties(data, benchmarkName) { + // Look at last sha hash. + const last = data[data.length - 1]; + return Object.keys(last[benchmarkName]); +} + +export function createColumns(data, benchmarkName) { + const varieties = getBenchmarkVarieties(data, benchmarkName); + return varieties.map(variety => [ + variety, ...data.map(d => { - const benchmark = d.benchmark[name]; - const meanValue = benchmark ? benchmark.mean : 0; - return meanValue || null; + if (d[benchmarkName] != null) { + if (d[benchmarkName][variety] != null) { + const v = d[benchmarkName][variety]; + if (benchmarkName == "benchmark") { + const meanValue = v ? v.mean : 0; + return meanValue || null; + } else { + return v; + } + } + } + return null; }) ]); } +export function createExecTimeColumns(data) { + return createColumns(data, "benchmark"); +} + +export function createThroughputColumns(data) { + return createColumns(data, "throughput"); +} + export function createBinarySizeColumns(data) { const propName = "binary_size"; const binarySizeNames = Object.keys(data[data.length - 1][propName]); @@ -108,6 +131,7 @@ export async function main() { const travisData = (await getTravisData()).filter(d => d.duration > 0); const execTimeColumns = createExecTimeColumns(data); + const throughputColumns = createThroughputColumns(data); const binarySizeColumns = createBinarySizeColumns(data); const threadCountColumns = createThreadCountColumns(data); const syscallCountColumns = createSyscallCountColumns(data); @@ -146,6 +170,24 @@ export async function main() { } }); + c3.generate({ + bindto: "#throughput-chart", + data: { + columns: throughputColumns, + onclick: viewCommitOnClick(sha1List) + }, + axis: { + x: { + type: "category", + show: false, + categories: sha1ShortList + }, + y: { + label: "seconds" + } + } + }); + c3.generate({ bindto: "#binary-size-chart", data: { diff --git a/website/app_test.js b/website/app_test.js index 029f659f3c82f3..42891bf6bef7f5 100644 --- a/website/app_test.js +++ b/website/app_test.js @@ -22,6 +22,12 @@ const regularData = [ "main.js.map": 80000000, "snapshot_deno.bin": 70000000 }, + throughput: { + "100M_tcp": 3.6, + "100M_cat": 3.0, + "10M_tcp": 1.6, + "10M_cat": 1.0 + }, benchmark: { hello: { mean: 0.05 @@ -54,6 +60,12 @@ const regularData = [ "main.js.map": 80000001, "snapshot_deno.bin": 70000001 }, + throughput: { + "100M_tcp": 3.6, + "100M_cat": 3.0, + "10M_tcp": 1.6, + "10M_cat": 1.0 + }, benchmark: { hello: { mean: 0.055 @@ -84,6 +96,7 @@ const irregularData = [ created_at: "2018-01-01T01:00:00Z", sha1: "123", benchmark: {}, + throughput: {}, binary_size: {}, thread_count: {}, syscall_count: {} @@ -97,6 +110,9 @@ const irregularData = [ cold_hello: {}, cold_relative_import: {} }, + throughput: { + "100M_tcp": 3.0 + }, binary_size: { deno: 1 }, diff --git a/website/index.html b/website/index.html index 84291e5d891c1f..03bb268bf0248f 100644 --- a/website/index.html +++ b/website/index.html @@ -26,6 +26,9 @@

Execution time

A cold startup is when deno must compile from scratch.
+

Throughput

+
+

Executable size

deno ships only a single binary. We track its size here.