diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000..f142c41 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,28 @@ +name: Benchmarks - stargate-mongoose +on: + pull_request: + push: +jobs: + benchmarks: + runs-on: ubuntu-latest + steps: + - name: disable and stop mono-xsp4.service + run: | + sudo kill -9 $(sudo lsof -t -i:8084) + - uses: actions/checkout@v3 + - name: Set up JSON API + run: | + chmod +x bin/start_json_api.sh + bin/start_json_api.sh + - name: Wait for services + run: | + while ! nc -z localhost 8181; do sleep 1; done + while ! nc -z localhost 8081; do sleep 1; done + - name: Setting up the node version + uses: actions/setup-node@v3 + with: + node-version: 16.15.0 + - name: setup project + run: npm i + - name: benchmarks + run: npm run benchmarks \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0b1bd25..b13d5ba 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ node_modules .vscode dist package-lock.json -.idea \ No newline at end of file +.idea +*.out +*.tgz \ No newline at end of file diff --git a/.npmignore b/.npmignore index bc70c5d..ce2e800 100644 --- a/.npmignore +++ b/.npmignore @@ -7,4 +7,6 @@ tests .env.example .github -*.tgz \ No newline at end of file +*.tgz + +benchmarks \ No newline at end of file diff --git a/benchmarks/.env.benchmark b/benchmarks/.env.benchmark new file mode 100644 index 0000000..55fbd5d --- /dev/null +++ b/benchmarks/.env.benchmark @@ -0,0 +1,4 @@ +JSON_API_URI=http://127.0.0.1:8181/v1/benchmark +JSON_API_USERNAME=cassandra +JSON_API_PASSWORD=cassandra +JSON_API_AUTH_URL=http://127.0.0.1:8081/v1/auth \ No newline at end of file diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..edaf405 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,95 @@ +# Benchmarks + +Each `benchmark-*.ts` file is a separate benchmark that tests different functionality. +Benchmarks use either stargate-mongoose or the Axios HTTP client directly; this is to check whether stargate-mongoose is adding significant overhead or not. + +## Running Benchmarks + +The easiest way to run all benchmarks is using the following npm command: + +``` +npm run benchmarks +``` + +This will run all benchmarks, and output results to both stdout and `.out` files in the `benchmarks` directory. + +You can also run benchmarks individually using `ts-node` as follows: + +``` +$ ./node_modules/.bin/ts-node -r ./benchmarks/setup.ts ./benchmarks/benchmark-findone-mongoose.ts +{ + "name": "benchmark-findone-mongoose", + "totalTimeMS": 14944 +} +``` + +The `benchmarks/.env.benchmark` contains environment variables that configure the connection details for the benchmarks. +The `benchmarks/setup.ts` file sets up the environment using [dotenv](https://npmjs.com/package/dotenv). + +## Astra Benchmarks + +There are 3 benchmarks that connect to Astra, rather than local JSON API: + +* `benchmark-findone-axios-vector-astra.ts` +* `benchmark-findone-mongoose-vector-astra.ts` +* `benchmark-findone-mongoose-vector-astra-http2.ts` + +By default, these benchmarks are skipped: + +``` +$ ./node_modules/.bin/ts-node -r ./benchmarks/setup.ts ./benchmarks/benchmark-findone-mongoose-vector-astra +{"name":"benchmark-findone-mongoose-vector-astra"} +``` + +To run these benchmarks, you need to add the following environment variables to `.env.benchmark`: + +``` +ASTRA_CONNECTION_STRING= +ASTRA_COLLECTION_NAME= +``` + +`ASTRA_CONNECTION_STRING` contains the connection string that will be passed to `mongoose.connect()`, like `https://abc-us-east-2.apps.astra.datastax.com/api/json/v1/mykeyspace?applicationToken=AstraCS:...`, and `ASTRA_COLLECTION_NAME` is the name of the collection that the Mongoose model will connect to. +The Astra benchmarks assume that there is already a collection with data set up in Astra. + +With these environment variables set up, you can run the Astra benchmarks: + +``` +$ ./node_modules/.bin/ts-node -r ./benchmarks/setup.ts ./benchmarks/benchmark-findone-mongoose-vector-astra +{ + "name": "benchmark-findone-mongoose-vector-astra", + "totalTimeMS": 20966 +} +``` + +If you add `ASTRA_CONNECTION_STRING` to `.env.benchmark`, be careful not to accidentally commit your changes to git. +An alternative approach is to set the environment variables in your CLI command as follows: + +``` +$ env ASTRA_CONNECTION_STRING="https://abc-us-east-2.apps.astra.datastax.com/..." env ASTRA_COLLECTION_NAME=mycollection ./node_modules/.bin/ts-node -r ./benchmarks/setup.ts ./benchmarks/benchmark-findone-mongoose-vector-astra +{ + "name": "benchmark-findone-mongoose-vector-astra", + "totalTimeMS": 20436 +} +``` + +## Pinecone Benchmarks + +First, install the Pinecone client: + +``` +npm install @pinecone-database/pinecone --no-save +``` + +Then set up sample data: + +``` +``` +env PINECONE_INDEX=your-index-here env PINECONE_API_KEY=your-api-key-here ./node_modules/.bin/ts-node benchmarks/setup-pinecone-vector-example.ts +``` +``` + +Then run Pinecone benchmarks: + +``` +env PINECONE_INDEX=your-index-here env PINECONE_API_KEY=your-api-key-here ./node_modules/.bin/ts-node benchmarks/benchmark-findone-pinecone.ts +``` \ No newline at end of file diff --git a/benchmarks/benchmark-findone-axios-vector-astra.ts b/benchmarks/benchmark-findone-axios-vector-astra.ts new file mode 100644 index 0000000..aaf17b7 --- /dev/null +++ b/benchmarks/benchmark-findone-axios-vector-astra.ts @@ -0,0 +1,76 @@ +import axios from 'axios'; +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + if (!process.env.ASTRA_CONNECTION_STRING) { + console.log('{"name":"benchmark-findone-axios-vector-astra"}'); + return; + } + await mongoose.connect(process.env.ASTRA_CONNECTION_STRING, { + isAstra: true + } as mongoose.ConnectOptions); + + const Vector = mongoose.model( + 'Vector', + new mongoose.Schema({ + $vector: { + type: [Number] + }, + prompt: { + type: String, + required: true + } + }, { + autoCreate: false + }), + process.env.ASTRA_COLLECTION_NAME + ); + + // @ts-ignore + const dbName = Vector.db.db.name; + + const collectionName = process.env.ASTRA_COLLECTION_NAME ?? ''; + + // @ts-ignore + const baseUrl = mongoose.connection.getClient().httpClient.baseUrl; + + // @ts-ignore + const token = mongoose.connection.getClient().httpClient.applicationToken; + const $meta = [1, ...Array(1535).fill(0)]; + const start = Date.now(); + for (let i = 0; i < 100; ++i) { + await axios.post( + `${baseUrl}/${dbName}/${collectionName}`, + { + findOne: { + filter: {}, + sort: { $vector: $meta } + } + }, + { + headers: { + Token: token + } + } + ); + } + const results = { + name: 'benchmark-findone-axios-vector-astra', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/benchmark-findone-axios-vector.ts b/benchmarks/benchmark-findone-axios-vector.ts new file mode 100644 index 0000000..577f4a1 --- /dev/null +++ b/benchmarks/benchmark-findone-axios-vector.ts @@ -0,0 +1,81 @@ +import axios from 'axios'; +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + await mongoose.connect(process.env.JSON_API_URI ?? '', { + username: process.env.JSON_API_USERNAME, + password: process.env.JSON_API_PASSWORD, + authUrl: process.env.JSON_API_AUTH_URL + } as mongoose.ConnectOptions); + const Vector = mongoose.model( + 'Vector', + new mongoose.Schema({ + $vector: { + type: [Number] + }, + prompt: { + type: String, + required: true + } + }, { + collectionOptions: { vector: { size: 1536, function: 'cosine' } }, + autoCreate: false + }), + 'vectors' + ); + + // @ts-ignore + const dbName = Vector.db.db.name; + + await Vector.db.dropCollection('vectors').catch(() => {}); + await Vector.createCollection(); + + const numVectors = 1000; + for (let i = 0; i < numVectors; ++i) { + const $vector = Array(1536).fill(0); + $vector[i] = 1; + await Vector.create({ $vector, prompt: `Test ${i}` }); + } + + // @ts-ignore + const baseUrl = mongoose.connection.getClient().httpClient.baseUrl; + // @ts-ignore + const token = mongoose.connection.getClient().httpClient.applicationToken; + const $meta = [1, ...Array(1535).fill(0)]; + const start = Date.now(); + for (let i = 0; i < 100; ++i) { + await axios.post( + `${baseUrl}/${dbName}/vectors`, + { + findOne: { + filter: {}, + sort: { $vector: $meta } + } + }, + { + headers: { + Token: token + } + } + ); + } + const results = { + name: 'benchmark-findone-axios-vector', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/benchmark-findone-axios.ts b/benchmarks/benchmark-findone-axios.ts new file mode 100644 index 0000000..044de1f --- /dev/null +++ b/benchmarks/benchmark-findone-axios.ts @@ -0,0 +1,75 @@ +import axios from 'axios'; +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + const uri = process.env.JSON_API_URI ?? ''; + await mongoose.connect(uri, { + username: process.env.JSON_API_USERNAME, + password: process.env.JSON_API_PASSWORD, + authUrl: process.env.JSON_API_AUTH_URL + } as mongoose.ConnectOptions); + const Test = mongoose.model('Test', new mongoose.Schema({ + name: { + type: String, + required: true + }, + email: { + type: String, + required: true + }, + age: { + type: Number + } + })); + + await Test.db.dropCollection('tests').catch(() => {}); + await Test.createCollection(); + + await Test.create({ + name: 'John Smith', + email: 'john@gmail.com', + age: 30 + }); + + // @ts-ignore + const token = mongoose.connection.getClient().httpClient.applicationToken; + + const start = Date.now(); + for (let i = 0; i < 10000; ++i) { + await axios.post( + `${uri}/tests`, + { + findOne: { + filter: { + name: 'John Smith' + }, + options: {} + } + }, + { + headers: { + Token: token + } + } + ); + } + const results = { + name: 'benchmark-findone-axios', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/benchmark-findone-mongoose-vector-astra.ts b/benchmarks/benchmark-findone-mongoose-vector-astra.ts new file mode 100644 index 0000000..10229fe --- /dev/null +++ b/benchmarks/benchmark-findone-mongoose-vector-astra.ts @@ -0,0 +1,53 @@ +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + if (!process.env.ASTRA_CONNECTION_STRING) { + console.log('{"name":"benchmark-findone-mongoose-vector-astra"}'); + return; + } + await mongoose.connect(process.env.ASTRA_CONNECTION_STRING, { + isAstra: true + } as mongoose.ConnectOptions); + const Vector = mongoose.model( + 'Vector', + new mongoose.Schema({ + $vector: { + type: [Number] + }, + prompt: { + type: String, + required: true + } + }, { + autoCreate: false + }), + process.env.ASTRA_COLLECTION_NAME + ); + + const $meta = [1, ...Array(1535).fill(0)]; + const start = Date.now(); + for (let i = 0; i < 100; ++i) { + await Vector + .findOne({}) + .sort({ $vector: { $meta } }); + } + const results = { + name: 'benchmark-findone-mongoose-vector-astra', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/benchmark-findone-mongoose-vector.ts b/benchmarks/benchmark-findone-mongoose-vector.ts new file mode 100644 index 0000000..8dd95d7 --- /dev/null +++ b/benchmarks/benchmark-findone-mongoose-vector.ts @@ -0,0 +1,62 @@ +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + await mongoose.connect(process.env.JSON_API_URI ?? '', { + username: process.env.JSON_API_USERNAME, + password: process.env.JSON_API_PASSWORD, + authUrl: process.env.JSON_API_AUTH_URL + } as mongoose.ConnectOptions); + const Vector = mongoose.model( + 'Vector', + new mongoose.Schema({ + $vector: { + type: [Number] + }, + prompt: { + type: String, + required: true + } + }, { + collectionOptions: { vector: { size: 1536, function: 'cosine' } }, + autoCreate: false + }), + 'vectors' + ); + + await Vector.db.dropCollection('vectors').catch(() => {}); + await Vector.createCollection(); + + const numVectors = 1000; + for (let i = 0; i < numVectors; ++i) { + const $vector = Array(1536).fill(0); + $vector[i] = 1; + await Vector.create({ $vector, prompt: `Test ${i}` }); + } + + const $meta = [1, ...Array(1535).fill(0)]; + const start = Date.now(); + for (let i = 0; i < 100; ++i) { + await Vector + .findOne({}) + .sort({ $vector: { $meta } }); + } + const results = { + name: 'benchmark-findone-mongoose-vector', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/benchmark-findone-mongoose.ts b/benchmarks/benchmark-findone-mongoose.ts new file mode 100644 index 0000000..91b5352 --- /dev/null +++ b/benchmarks/benchmark-findone-mongoose.ts @@ -0,0 +1,55 @@ +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + await mongoose.connect(process.env.JSON_API_URI ?? '', { + username: process.env.JSON_API_USERNAME, + password: process.env.JSON_API_PASSWORD, + authUrl: process.env.JSON_API_AUTH_URL + } as mongoose.ConnectOptions); + const Test = mongoose.model('Test', new mongoose.Schema({ + name: { + type: String, + required: true + }, + email: { + type: String, + required: true + }, + age: { + type: Number + } + })); + + await Test.db.dropCollection('tests').catch(() => {}); + await Test.createCollection(); + + await Test.create({ + name: 'John Smith', + email: 'john@gmail.com', + age: 30 + }); + + const start = Date.now(); + for (let i = 0; i < 10000; ++i) { + await Test.findOne({ name: 'John Smith' }); + } + const results = { + name: 'benchmark-findone-mongoose', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/benchmark-findone-pinecone.ts b/benchmarks/benchmark-findone-pinecone.ts new file mode 100644 index 0000000..6347d71 --- /dev/null +++ b/benchmarks/benchmark-findone-pinecone.ts @@ -0,0 +1,36 @@ +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + let Pinecone; + try { + Pinecone = require('@pinecone-database/pinecone').Pinecone; + } catch (err) { + console.log('{"name":"benchmark-findone-pinecone"}'); + return; + } + + if (!process.env.PINECONE_API_KEY || !process.env.PINECONE_INDEX) { + console.log('{"name":"benchmark-findone-pinecone"}'); + return; + } + + const pinecone = new Pinecone(); + + const vector = [1, ...Array(1535).fill(0)]; + const index = process.env.PINECONE_INDEX; + const start = Date.now(); + for (let i = 0; i < 100; ++i) { + await pinecone.index(index).query({ topK: 1, vector, includeValues: true }); + } + const results = { + name: 'benchmark-findone-pinecone', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/benchmark-insert-axios.ts b/benchmarks/benchmark-insert-axios.ts new file mode 100644 index 0000000..961179c --- /dev/null +++ b/benchmarks/benchmark-insert-axios.ts @@ -0,0 +1,68 @@ +import axios from 'axios'; +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + await mongoose.connect(process.env.JSON_API_URI ?? '', { + username: process.env.JSON_API_USERNAME, + password: process.env.JSON_API_PASSWORD, + authUrl: process.env.JSON_API_AUTH_URL + } as mongoose.ConnectOptions); + const Test = mongoose.model('Test', new mongoose.Schema({ + name: { + type: String, + required: true + }, + email: { + type: String, + required: true + }, + age: { + type: Number + } + })); + + await Test.db.dropCollection('tests').catch(() => {}); + await Test.createCollection(); + // @ts-ignore + const token = mongoose.connection.getClient().httpClient.applicationToken; + + const start = Date.now(); + for (let i = 0; i < 10000; ++i) { + await axios.post( + `${process.env.JSON_API_URI}/tests`, + { + insertOne: { + document: { + name: `John Smith ${i}`, + email: `john${i}@gmail.com`, + age: 30 + } + } + }, + { + headers: { + Token: token + } + } + ); + } + const results = { + name: 'benchmark-insert-axios', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/benchmark-insert-mongoose.ts b/benchmarks/benchmark-insert-mongoose.ts new file mode 100644 index 0000000..538a314 --- /dev/null +++ b/benchmarks/benchmark-insert-mongoose.ts @@ -0,0 +1,53 @@ +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => process.exit(0), + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + await mongoose.connect(process.env.JSON_API_URI ?? '', { + username: process.env.JSON_API_USERNAME, + password: process.env.JSON_API_PASSWORD, + authUrl: process.env.JSON_API_AUTH_URL + } as mongoose.ConnectOptions); + const Test = mongoose.model('Test', new mongoose.Schema({ + name: { + type: String, + required: true + }, + email: { + type: String, + required: true + }, + age: { + type: Number + } + })); + + await Test.db.dropCollection('tests').catch(() => {}); + await Test.createCollection(); + + const start = Date.now(); + for (let i = 0; i < 10000; ++i) { + await Test.create({ + name: `John Smith ${i}`, + email: `john${i}@gmail.com`, + age: 30 + }); + } + const results = { + name: 'benchmark-insert-mongoose', + totalTimeMS: Date.now() - start + }; + console.log(JSON.stringify(results, null, ' ')); +} diff --git a/benchmarks/setup-astra-vector-example.ts b/benchmarks/setup-astra-vector-example.ts new file mode 100644 index 0000000..915517d --- /dev/null +++ b/benchmarks/setup-astra-vector-example.ts @@ -0,0 +1,73 @@ +import { driver } from '../'; +import mongoose from 'mongoose'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + if (!process.env.ASTRA_CONNECTION_STRING) { + throw new Error('Must set ASTRA_CONNECTION_STRING'); + } + if (!process.env.ASTRA_COLLECTION_NAME) { + throw new Error('Must set ASTRA_COLLECTION_NAME'); + } + await mongoose.connect(process.env.ASTRA_CONNECTION_STRING, { + isAstra: true, + useHTTP2: false + } as mongoose.ConnectOptions); + + const Vector = mongoose.model( + 'Vector', + new mongoose.Schema({ + $vector: { + type: [Number] + }, + prompt: { + type: String, + required: true + } + }, { + collectionOptions: { vector: { dimension: 1536, metric: 'cosine' } }, + autoCreate: false + }), + process.env.ASTRA_COLLECTION_NAME + ); + + console.log('Recreating collection...'); + await Vector.db.dropCollection(process.env.ASTRA_COLLECTION_NAME); + await Vector.createCollection(); + + console.log('Creating vectors...'); + const numVectors = 1000; + const start = Date.now(); + for (let i = 0; i < numVectors; ++i) { + console.log(`${i} / ${numVectors}`); + const $vector = Array(1536).fill(0); + $vector[i] = 1; + for (let j = 0; j < 3; ++j) { + try { + await Vector.create({ $vector, prompt: `Test ${i}` }); + break; + } catch (err) { + if (j >= 2) { + throw err; + } + await new Promise(resolve => setTimeout(resolve, 100)); + } + } + + } + + console.log('Done', Date.now() - start); + process.exit(0); +} \ No newline at end of file diff --git a/benchmarks/setup-pinecone-vector-example.ts b/benchmarks/setup-pinecone-vector-example.ts new file mode 100644 index 0000000..205df89 --- /dev/null +++ b/benchmarks/setup-pinecone-vector-example.ts @@ -0,0 +1,38 @@ +import { Pinecone } from '@pinecone-database/pinecone'; + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + if (!process.env.PINECONE_API_KEY || !process.env.PINECONE_INDEX) { + throw new Error('Cannot run pinecone setup without an API key'); + } + + const pinecone = new Pinecone(); + + console.log('Creating vectors...'); + + const numVectors = 1000; + const index = process.env.PINECONE_INDEX; + const start = Date.now(); + for (let i = 0; i < numVectors; ++i) { + console.log(`${i} / ${numVectors}`); + const values: number[] = Array(1536).fill(0); + values[i] = 1; + await pinecone.index(index).upsert([{ + id: '' + i, + values, + metadata: { + prompt: `Test ${i}` + } + }]); + } + + console.log('Done', Date.now() - start); + process.exit(0); +} \ No newline at end of file diff --git a/benchmarks/setup.ts b/benchmarks/setup.ts new file mode 100644 index 0000000..f569006 --- /dev/null +++ b/benchmarks/setup.ts @@ -0,0 +1,6 @@ +import dotenv from 'dotenv'; +import path from 'path'; + +dotenv.config({ + path: path.join('benchmarks', '.env.benchmark') +}); \ No newline at end of file diff --git a/benchmarks/upload-results.ts b/benchmarks/upload-results.ts new file mode 100644 index 0000000..282bb32 --- /dev/null +++ b/benchmarks/upload-results.ts @@ -0,0 +1,65 @@ +import { driver } from '../'; +import { execSync } from 'child_process'; +import fs from 'fs'; +import mongoose from 'mongoose'; +import path from 'path'; + +mongoose.set('autoCreate', false); +mongoose.set('autoIndex', false); + +mongoose.setDriver(driver); + +main().then( + () => { process.exit(0); }, + err => { + console.error(err); + process.exit(-1); + } +); + +async function main() { + if (!process.env.ASTRA_UPLOAD_CONNECTION_STRING) { + return; + } + + const githash = execSync('git rev-parse HEAD').toString().trim(); + + await mongoose.connect(process.env.ASTRA_UPLOAD_CONNECTION_STRING, { + isAstra: true + } as mongoose.ConnectOptions); + const BenchmarkResult = mongoose.model( + 'BenchmarkResult', + new mongoose.Schema({ + githash: { + type: String, + required: true + }, + name: { + type: String, + required: true + }, + totalTimeMS: { + type: Number + } + }, { autoCreate: true, timestamps: true }) + ); + await BenchmarkResult.init(); + + const files = fs + .readdirSync(path.join('.', 'benchmarks')) + .filter(file => file.endsWith('.out')); + for (const file of files) { + const { name, totalTimeMS } = JSON.parse( + fs.readFileSync(path.join('.', 'benchmarks', file), 'utf8') + ); + if (totalTimeMS == null) { + continue; + } + const doc = await BenchmarkResult.findOneAndUpdate( + { githash, name }, + { totalTimeMS }, + { upsert: true, returnDocument: 'after' } + ); + console.log(doc); + } +} \ No newline at end of file diff --git a/package.json b/package.json index 5f4f878..d9e5a76 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "url": "git+https://github.com/stargate/stargate-mongoose.git" }, "scripts": { + "benchmarks": "npm run build; for file in ./benchmarks/benchmark-*.ts; do echo \"$file\"; ts-node -r ./benchmarks/setup \"$file\" > \"$file\".out; cat \"$file\".out; done", "lint": "eslint .", "test": "env TEST_DOC_DB=jsonapi ts-mocha --forbid-only --paths -p tsconfig.json tests/**/*.test.ts", "test-astra": "env TEST_DOC_DB=astra nyc ts-mocha --forbid-only --paths -p tsconfig.json tests/**/*.test.ts",