diff --git a/Makefile b/Makefile index 3278667fb97..bd489d8b906 100644 --- a/Makefile +++ b/Makefile @@ -40,11 +40,6 @@ help: @echo ' make update.chart-entities update the charts_x_entities join table' @echo ' make reindex reindex (or initialise) search in Algolia' @echo ' make bench.search run search benchmarks' - @echo - @echo ' OPS (staff-only)' - @echo ' make deploy Deploy your local site to production' - @echo ' make stage Deploy your local site to staging' - @echo up: export DEBUG = 'knex:query' @@ -215,34 +210,6 @@ tmp-downloads/owid_metadata.sql.gz: @echo '==> Downloading metadata' ./devTools/docker/download-grapher-metadata-mysql.sh -deploy: node_modules - @echo '==> Starting from a clean slate...' - rm -rf itsJustJavascript - - @echo '==> Building...' - yarn lerna run build --skip-nx-cache - yarn run tsc -b - - @echo '==> Deploying...' - yarn buildAndDeploySite live - -stage: node_modules - @if [[ ! "$(STAGING)" ]]; then \ - echo 'ERROR: must set the staging environment'; \ - echo ' e.g. STAGING=halley make stage'; \ - exit 1; \ - fi - @echo '==> Preparing to deploy to $(STAGING)' - @echo '==> Starting from a clean slate...' - rm -rf itsJustJavascript - - @echo '==> Building...' - yarn lerna run build - yarn run tsc -b - - @echo '==> Deploying to $(STAGING)...' - yarn buildAndDeploySite $(STAGING) - test: node_modules @echo '==> Linting' yarn run eslint diff --git a/baker/DeployTarget.ts b/baker/DeployTarget.ts deleted file mode 100644 index 4e981228c74..00000000000 --- a/baker/DeployTarget.ts +++ /dev/null @@ -1,19 +0,0 @@ -export enum DeployTarget { - live = "live", - staging = "staging", - hans = "hans", - playfair = "playfair", - jefferson = "jefferson", - nightingale = "nightingale", - explorer = "explorer", - exemplars = "exemplars", - tufte = "tufte", - roser = "roser", - snow = "snow", - halley = "halley", - neurath = "neurath", - ptolemy = "ptolemy", - kelley = "kelley", -} - -export const ProdTarget = "live" diff --git a/baker/Deployer.ts b/baker/Deployer.ts deleted file mode 100644 index 0c844439159..00000000000 --- a/baker/Deployer.ts +++ /dev/null @@ -1,314 +0,0 @@ -import fs from "fs-extra" -import { prompt } from "prompts" -import ProgressBar from "progress" -import { execWrapper } from "../db/execWrapper.js" -import { spawn } from "child_process" -import { simpleGit, SimpleGit } from "simple-git" -import { WriteStream } from "tty" -import { ProgressStream } from "./ProgressStream.js" -import { DeployTarget, ProdTarget } from "./DeployTarget.js" - -const TEMP_DEPLOY_SCRIPT_PREFIX = `tempDeployScript.` - -interface DeployerOptions { - owidGrapherRootDir: string - userRunningTheDeploy: string - target: DeployTarget - skipChecks?: boolean - runChecksRemotely?: boolean -} - -const OWID_STAGING_DROPLET_IP = "165.22.127.239" -const OWID_LIVE_DROPLET_IP = "209.97.185.49" - -export class Deployer { - private options: DeployerOptions - private progressBar: ProgressBar - private stream: ProgressStream - constructor(options: DeployerOptions) { - this.options = options - const { target, skipChecks, runChecksRemotely } = this.options - - this.stream = new ProgressStream(process.stderr) - // todo: a smarter way to precompute out the number of steps? - const testSteps = !skipChecks && !runChecksRemotely ? 1 : 0 - this.progressBar = new ProgressBar( - `Baking and deploying to ${target} [:bar] :current/:total :elapseds :name\n`, - { - total: 21 + testSteps, - renderThrottle: 0, // print on every tick - stream: this.stream as unknown as WriteStream, - } - ) - } - - private async runAndTick(command: string) { - await execWrapper(command) - this.progressBar.tick({ name: `โœ… finished ${command}` }) - } - - private get isValidTarget() { - return new Set(Object.values(DeployTarget)).has(this.options.target) - } - - get targetIsProd() { - return this.options.target === ProdTarget - } - - private get targetIpAddress() { - return this.targetIsProd - ? OWID_LIVE_DROPLET_IP - : OWID_STAGING_DROPLET_IP - } - - // todo: I have not tested this yet, and would be surprised if it worked on the first attempt. - private async runPreDeployChecksRemotely() { - const { owidGrapherRootDir } = this.options - const { rsyncTargetDirForTests } = this.pathsOnTarget - const RSYNC_TESTS = `rsync -havz --no-perms --progress --delete --include=/test --include=*.test.ts --include=*.test.tsx --exclude-from=${owidGrapherRootDir}/.rsync-ignore` - await execWrapper( - `${RSYNC_TESTS} ${owidGrapherRootDir} ${this.sshHost}:${rsyncTargetDirForTests}` - ) - - const script = `cd ${rsyncTargetDirForTests} -yarn install --immutable -yarn testPrettierAll` - await execWrapper(`ssh -t ${this.sshHost} 'bash -e -s' ${script}`) - - this.progressBar.tick({ - name: "โœ…๐Ÿ“ก finished running predeploy checks remotely", - }) - } - - private async runLiveSafetyChecks() { - const { simpleGit } = this - const branches = await simpleGit.branchLocal() - const branch = await branches.current - if (branch !== "master") - this.printAndExit( - "To deploy to live please run from the master branch." - ) - - // Making sure we have the latest changes from the upstream - // Also, will fail if working copy is not clean - try { - const gitStatus = await simpleGit.status() - // gitStatus.isClean() checks for staged, unstaged, and untracked files - if (!gitStatus.isClean()) throw "Git working directory is not clean" - - await simpleGit.pull("origin", undefined, { "--rebase": "true" }) - } catch (err) { - this.printAndExit(JSON.stringify(err)) - } - - const response = await prompt({ - type: "confirm", - name: "confirmed", - message: "Are you sure you want to deploy to live?", - }) - if (!response?.confirmed) this.printAndExit("Cancelled") - } - - private _simpleGit?: SimpleGit - private get simpleGit() { - if (!this._simpleGit) - this._simpleGit = simpleGit({ - baseDir: this.options.owidGrapherRootDir, - binary: "git", - maxConcurrentProcesses: 1, - }) - return this._simpleGit - } - - private get pathsOnTarget() { - const { target, userRunningTheDeploy } = this.options - const owidUserHomeDir = "/home/owid" - const owidUserHomeTmpDir = `${owidUserHomeDir}/tmp` - - return { - owidUserHomeDir, - owidUserHomeTmpDir, - rsyncTargetDir: `${owidUserHomeTmpDir}/${target}-${userRunningTheDeploy}`, - rsyncTargetDirTmp: `${owidUserHomeTmpDir}/${target}-${userRunningTheDeploy}-tmp`, - rsyncTargetDirForTests: `${owidUserHomeTmpDir}/${target}-tests`, - finalTargetDir: `${owidUserHomeDir}/${target}`, - oldRepoBackupDir: `${owidUserHomeTmpDir}/${target}-old`, - finalDataDir: `${owidUserHomeDir}/${target}-data`, - } - } - - private get sshHost() { - return `owid@${this.targetIpAddress}` - } - - private async writeHeadDotText() { - const { simpleGit } = this - const { owidGrapherRootDir } = this.options - const gitCommitSHA = await simpleGit.revparse(["HEAD"]) - - // Write the current commit SHA to public/head.txt so we always know which commit is deployed - fs.writeFileSync( - owidGrapherRootDir + "/public/head.txt", - gitCommitSHA, - "utf8" - ) - this.progressBar.tick({ name: "โœ… finished writing head.txt" }) - } - - // ๐Ÿ“ก indicates that a task is running/ran on the remote server - async buildAndDeploy() { - const { skipChecks, runChecksRemotely } = this.options - - if (this.targetIsProd) await this.runLiveSafetyChecks() - else if (!this.isValidTarget) - this.printAndExit( - "Please select either live or a valid test target." - ) - - this.progressBar.tick({ - name: "โœ… finished validating deploy arguments", - }) - - // make sure that no old assets are left over from an old deploy - await this.runAndTick(`yarn cleanTsc`) - await this.runAndTick(`yarn buildTsc`) - - if (runChecksRemotely) await this.runPreDeployChecksRemotely() - else if (skipChecks) { - if (this.targetIsProd) - this.printAndExit(`Cannot skip checks when deploying to live`) - this.progressBar.tick({ - name: "โœ… finished checks because we skipped them", - }) - } else { - await this.runAndTick(`yarn testJest`) - } - - await this.writeHeadDotText() - await this.ensureTmpDirExistsOnServer() - - await this.generateShellScriptsAndRunThemOnServer() - - this.progressBar.tick({ - name: `โœ… ๐Ÿ“ก finished everything`, - }) - this.stream.replay() - } - - // todo: the old deploy script would generete BASH on the fly and run it on the server. we should clean that up and remove these shell scripts. - private async generateShellScriptsAndRunThemOnServer(): Promise { - const { target, owidGrapherRootDir } = this.options - - const { - rsyncTargetDirTmp, - finalTargetDir, - rsyncTargetDir, - oldRepoBackupDir, - finalDataDir, - } = this.pathsOnTarget - - const scripts: any = { - clearOldTemporaryRepo: `rm -rf ${rsyncTargetDirTmp}`, - copySyncedRepo: `cp -r ${rsyncTargetDir} ${rsyncTargetDirTmp}`, // Copy the synced repo-- this is because we're about to move it, and we want the original target to stay around to make future syncs faster - createDataSoftlinks: `mkdir -p ${finalDataDir}/bakedSite && ln -sf ${finalDataDir}/bakedSite ${rsyncTargetDirTmp}/bakedSite`, - createDatasetSoftlinks: `mkdir -p ${finalDataDir}/datasetsExport && ln -sf ${finalDataDir}/datasetsExport ${rsyncTargetDirTmp}/datasetsExport`, - createSettingsSoftlinks: `ln -sf ${finalDataDir}/.env ${rsyncTargetDirTmp}/.env`, - yarn: `cd ${rsyncTargetDirTmp} && yarn install --immutable`, - lernaBuild: `cd ${rsyncTargetDirTmp} && yarn lerna run build`, - vite: `cd ${rsyncTargetDirTmp} && yarn buildVite`, - migrateDb: `cd ${rsyncTargetDirTmp} && yarn runDbMigrations`, - algolia: `cd ${rsyncTargetDirTmp} && node --enable-source-maps --unhandled-rejections=strict itsJustJavascript/baker/algolia/configureAlgolia.js`, - createQueueFile: `cd ${rsyncTargetDirTmp} && touch .queue && chmod 0666 .queue`, - swapFolders: `rm -rf ${oldRepoBackupDir} && mv ${finalTargetDir} ${oldRepoBackupDir} || true && mv ${rsyncTargetDirTmp} ${finalTargetDir}`, - restartAdminServer: `pm2 restart ${target}`, - restartDeployQueueServer: `pm2 restart ${target}-deploy-queue`, - } - - Object.keys(scripts).forEach((name) => { - const localPath = `${owidGrapherRootDir}/${TEMP_DEPLOY_SCRIPT_PREFIX}${name}.sh` - fs.writeFileSync(localPath, scripts[name], "utf8") - fs.chmodSync(localPath, "755") - }) - - await this.copyLocalRepoToServerTmpDirectory() - - for await (const name of Object.keys(scripts)) { - await this.runAndStreamScriptOnRemoteServerViaSSH( - `${rsyncTargetDir}/${TEMP_DEPLOY_SCRIPT_PREFIX}${name}.sh` - ) - const localPath = `${owidGrapherRootDir}/${TEMP_DEPLOY_SCRIPT_PREFIX}${name}.sh` - fs.removeSync(localPath) - } - } - - printAndExit(message: string) { - // eslint-disable-next-line no-console - console.log(message) - process.exit() - } - - private async ensureTmpDirExistsOnServer() { - const { sshHost } = this - const { owidUserHomeTmpDir } = this.pathsOnTarget - await execWrapper(`ssh ${sshHost} mkdir -p ${owidUserHomeTmpDir}`) - this.progressBar.tick({ - name: `โœ… ๐Ÿ“ก finished ensuring ${owidUserHomeTmpDir} exists on ${sshHost}`, - }) - } - - private async copyLocalRepoToServerTmpDirectory() { - const { owidGrapherRootDir } = this.options - const { rsyncTargetDir } = this.pathsOnTarget - const RSYNC = `rsync -havz --no-perms --progress --delete --delete-excluded --prune-empty-dirs --exclude-from=${owidGrapherRootDir}/.rsync-ignore` - await execWrapper( - `${RSYNC} ${owidGrapherRootDir}/ ${this.sshHost}:${rsyncTargetDir}` - ) - this.progressBar.tick({ - name: `โœ… ๐Ÿ“ก finished rsync of ${owidGrapherRootDir} to ${this.sshHost} ${rsyncTargetDir}`, - }) - } - - private async runAndStreamScriptOnRemoteServerViaSSH( - path: string - ): Promise { - // eslint-disable-next-line no-console - console.log(`๐Ÿ“ก Running ${path} on ${this.sshHost}`) - const bashTerminateIfAnyNonZero = "bash -e" // https://stackoverflow.com/questions/9952177/whats-the-meaning-of-the-parameter-e-for-bash-shell-command-line/9952249 - const pseudoTty = "-tt" // https://stackoverflow.com/questions/7114990/pseudo-terminal-will-not-be-allocated-because-stdin-is-not-a-terminal - const params = [ - pseudoTty, - this.sshHost, - bashTerminateIfAnyNonZero, - path, - ] - const child = spawn(`ssh`, params) - - child.stdout.on("data", (data) => { - const trimmed = data.toString().trim() - if (!trimmed) return - // eslint-disable-next-line no-console - console.log(trimmed) - }) - - child.stderr.on("data", (data) => { - const trimmed = data.toString().trim() - if (!trimmed) return - // eslint-disable-next-line no-console - console.error(trimmed) - }) - - const exitCode: number = await new Promise((resolve) => { - child.on("close", resolve) - }) - - if (exitCode !== 0) { - throw new Error( - `๐Ÿ“กโ›”๏ธ failed running ${path} [exit code ${exitCode}]` - ) - } - - this.progressBar.tick({ - name: `๐Ÿ“กโœ… finished running ${path}`, - }) - } -} diff --git a/baker/ProgressStream.ts b/baker/ProgressStream.ts deleted file mode 100644 index 65aa0a188aa..00000000000 --- a/baker/ProgressStream.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { WriteStream } from "tty" - -// Wrap stderr before passing it to ProgressBar so we can save all writes -// and replay them at the end of the bake. Without this the progress bar class -// works fine, but there is no way to show the summary once the job is complete. -export class ProgressStream implements Partial { - private wrappedStream: WriteStream - constructor(wrap: WriteStream) { - this.wrappedStream = wrap - } - - isTTY = true - - private allWrites: string[] = [] - - replay() { - console.log(this.allWrites.join("")) - } - - write(buffer: string) { - this.allWrites.push(buffer) - return this.wrappedStream.write(buffer) - } - - cursorTo(index: number) { - return this.wrappedStream.cursorTo(index) - } - - clearLine(direction: 1) { - return this.wrappedStream.clearLine(direction) - } - - get columns() { - return this.wrappedStream.columns - } -} diff --git a/baker/buildAndDeploySite.ts b/baker/buildAndDeploySite.ts deleted file mode 100755 index 3852ebf59d9..00000000000 --- a/baker/buildAndDeploySite.ts +++ /dev/null @@ -1,35 +0,0 @@ -#! /usr/bin/env node - -import { Deployer } from "./Deployer.js" -import yargs from "yargs" -import { hideBin } from "yargs/helpers" -import os from "os" -import path from "path" -import { DeployTarget } from "./DeployTarget.js" - -void yargs(hideBin(process.argv)) - .command<{ - target: DeployTarget - skipChecks: boolean - runChecksRemotely: boolean - steps?: string[] - }>( - "$0 [target]", - "Deploy the site to a remote environment", - (yargs) => { - yargs.boolean(["skip-checks", "run-checks-remotely"]) - }, - async ({ target, skipChecks, runChecksRemotely }) => { - const deployer = new Deployer({ - target: target as any, - userRunningTheDeploy: os.userInfo().username, - owidGrapherRootDir: path.normalize(__dirname + "/../../"), - skipChecks, - runChecksRemotely: runChecksRemotely, - }) - await deployer.buildAndDeploy() - } - ) - .help() - .alias("help", "h") - .strict().argv diff --git a/package.json b/package.json index fc29b66b5c8..c94d7ebaf81 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,6 @@ "packageManager": "yarn@4.1.1", "scripts": { "batchTagWithGpt": "tsx --tsconfig tsconfig.tsx.json baker/batchTagWithGpt.ts", - "buildAndDeploySite": "tsx --tsconfig tsconfig.tsx.json baker/buildAndDeploySite.ts", "buildCoverage": "jest --coverage=true --coverageProvider=v8", "buildLocalBake": "tsx --tsconfig tsconfig.tsx.json baker/buildLocalBake.ts", "buildTsc": "tsc -b -verbose",