From aed071b922766e0b3dcf58c3ce821ad126bca589 Mon Sep 17 00:00:00 2001 From: Ozzie Vasdi <29984068+ovasdi@users.noreply.github.com> Date: Tue, 14 Sep 2021 16:40:41 -0400 Subject: [PATCH] feat: turn-key env (#2972) * turn-key setup inital pass - add setup script installing all requirements - add brewfile to install mongo and elasticsearch - edit env.example removing shared configs - update development compose file - pin node version 12.22.6 - update Dockerfile to use node 12.22 * implement loadenv.js * update README * update development file * escape SALT '$' char for mac dockerized dev to work * introduce .env.oss * proload loadenv instead of dotenv for task and webpack scripts * minor edits * edit docker-compose files * switch setup script to execute in bash, use tput for colors * fix a broken link in README * add a note about how to use .env --- .dockerignore | 3 +- .env.example | 52 ++------------------------- .env.oss | 30 ++++++++++++++++ .gitignore | 9 +++-- .nvmrc | 2 +- Brewfile | 4 +++ Dockerfile | 2 +- README.md | 79 ++++++++++------------------------------- hokusai/development.yml | 23 +++++++++--- hokusai/test.yml | 2 ++ package.json | 6 ++-- scripts/dev.sh | 2 +- scripts/setup.sh | 39 ++++++++++++++++++++ scripts/start.sh | 4 +-- src/index.js | 2 +- src/lib/loadenv.js | 33 +++++++++++++++++ 16 files changed, 165 insertions(+), 127 deletions(-) create mode 100644 .env.oss create mode 100644 Brewfile create mode 100755 scripts/setup.sh create mode 100644 src/lib/loadenv.js diff --git a/.dockerignore b/.dockerignore index 57f293d9e..87f22982e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -20,4 +20,5 @@ tmp node_modules npm-debug.log .vscode -.circleci \ No newline at end of file +.circleci +Brewfile* diff --git a/.env.example b/.env.example index ae7d593b0..e38f98fee 100644 --- a/.env.example +++ b/.env.example @@ -1,16 +1,5 @@ -NODE_ENV=development -PORT=3005 -APP_NAME=positron-staging -APP_URL=http://localhost:3005 -API_URL=http://localhost:3005/api -FORCE_URL=https://staging.artsy.net -ARTSY_URL=https://stagingapi.artsy.net -SESSION_SECRET=p0s1tr0n -ELASTICSEARCH_PORT=9200 - -# Local development only for the below; refers to the key in docker-compose.yml -ELASTICSEARCH_URL=http://localhost:9200 -MONGOHQ_URL=mongodb://localhost:27017/positron +# The .env.example file provides an *initial* .env file. +# Local overrides should go in the .env file. ## # Set the DATADOG_AGENT_HOSTNAME to some value and start a local agent via Docker @@ -37,40 +26,3 @@ MONGOHQ_URL=mongodb://localhost:27017/positron #``` ## DATADOG_AGENT_HOSTNAME=datadog-agent - -# Needed to link Reaction: `yarn link @artsy/reaction` -GRAPHQL_ENDPOINT=https://metaphysics-staging.artsy.net - -TECH_SUPPORT=craig@artsymail.com -DEBUG=api,client,app -SALT=$2a$10$PJrPMBadu1NPdmnshBgFbe -API_MAX=100 -API_PAGE_SIZE=10 -GEMINI_APP=writer-staging -FORCE_COLOR=1 -WEBFONT_URL=http://webfonts.artsy.net/ -IP_BLACKLIST= -NO_INDEX_CHANNELS= -DEV_FEATURES=1 - -# OSS version of the .env file -# Note: The ARTSY_ID & ARTSY_SECRET are known keys for Artsy OSS -# projects, and are not a problem. OSS people: Please don't abuse the keys, -# as then we'll have to change it, making it harder for others to learn from. -# As such, these keys do not come under the Artsy security bounty. - -ARTSY_ID=e750db60ac506978fc70 -ARTSY_SECRET=3a33d2085cbd1176153f99781bbce7c6 -EDITORIAL_CHANNEL=REPLACE_ME -SEGMENT_WRITE_KEY=REPLACE_ME -GRAVITY_CLOUDFRONT_URL=REPLACE_ME -FULCRUM_BUCKET=REPLACE_ME -S3_KEY=REPLACE_ME -S3_SECRET=REPLACE_ME -SAILTHRU_KEY=REPLACE_ME -SAILTHRU_SECRET=REPLACE_ME -SECURE_IMAGES_URL=REPLACE_ME -FB_PAGE_ID=REPLACE_ME -DEFAULT_PARTNER_ID=REPLACE_ME -SENTRY_PUBLIC_DSN=REPLACE_ME -SENTRY_PRIVATE_DSN=REPLACE_ME diff --git a/.env.oss b/.env.oss new file mode 100644 index 000000000..78c90bb6d --- /dev/null +++ b/.env.oss @@ -0,0 +1,30 @@ +# OSS version of the .env.shared file +# Note: The ARTSY_ID & ARTSY_SECRET are known keys for Artsy OSS +# projects, and are not a problem. OSS people: Please don't abuse the keys, +# as then we'll have to change it, making it harder for others to learn from. +# As such, these keys do not come under the Artsy security bounty. + +ARTSY_ID=e750db60ac506978fc70 +ARTSY_SECRET=3a33d2085cbd1176153f99781bbce7c6 +NODE_ENV=development +PORT=3005 +APP_NAME=positron-staging +APP_URL=http://localhost:3005 +API_URL=http://localhost:3005/api +FORCE_URL=https://staging.artsy.net +ARTSY_URL=https://stagingapi.artsy.net +SESSION_SECRET=p0s1tr0n +ELASTICSEARCH_PORT=9200 +ELASTICSEARCH_URL=http://localhost:9200 +MONGOHQ_URL=mongodb://localhost:27017/positron +GRAPHQL_ENDPOINT=https://metaphysics-staging.artsy.net # Needed to link Reaction: `yarn link @artsy/reaction` +DEBUG=api,client,app +SALT=$2a$10$PJrPMBadu1NPdmnshBgFbe +API_MAX=100 +API_PAGE_SIZE=10 +GEMINI_APP=writer-staging +FORCE_COLOR=1 +WEBFONT_URL=http://webfonts.artsy.net/ +IP_BLACKLIST= +NO_INDEX_CHANNELS= +DEV_FEATURES=1 diff --git a/.gitignore b/.gitignore index 681b82351..ca98d6af0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,7 @@ npm-debug.log .DS_Store dump.rdb .DS_Store -.env -.env.ignore +.env* .cache data @@ -28,3 +27,9 @@ coverage* .nyc_output/ cypress.env.json +Brewfile.lock.json + +# Do not ignore +!.env.example +!.env.test +!.env.oss diff --git a/.nvmrc b/.nvmrc index 48082f72f..9293735bb 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -12 +12.22.6 diff --git a/Brewfile b/Brewfile new file mode 100644 index 000000000..0e738bf84 --- /dev/null +++ b/Brewfile @@ -0,0 +1,4 @@ +tap 'artsy/formulas' +tap 'mongodb/brew' +brew 'mongodb-community@4.0', link: true, restart_service: true +brew 'elasticsearch@5.6', restart_service: true diff --git a/Dockerfile b/Dockerfile index ec8e71b8d..b0774dec8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12.18-alpine +FROM node:12.22-alpine ENV PORT 3005 EXPOSE 3005 diff --git a/README.md b/README.md index c494296b5..d4536c974 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - **State:** production - **Production:** [https://writer.artsy.net/](https://writer.artsy.net/) | [Kubernetes](https://kubernetes.prd.artsy.systems/#!/deployment/default/positron-web?namespace=default) -- **Staging:** [http://stagingwriter.artsy.net/](http://stagingwriter.artsy.net//) | [Kubernetes](https://kubernetes.stg.artsy.systems/#!/deployment/default/positron-web?namespace=default) +- **Staging:** [https://stagingwriter.artsy.net/](https://stagingwriter.artsy.net/) | [Kubernetes](https://kubernetes.stg.artsy.systems/#!/deployment/default/positron-web?namespace=default) - **Logs:** - [Production](https://papertrailapp.com/groups/3675843/events?q=host%3Apositron-web) - [Staging](https://papertrailapp.com/groups/3674473/events?q=host%3Apositron-web) @@ -18,62 +18,30 @@ - **MongoDB:** [Atlas](https://cloud.mongodb.com/v2/5be44a7aff7a254a8327cd3a#clusters) - **Github:** [https://github.com/artsy/positron/](https://github.com/artsy/positron/) - **CI:** [CircleCI](https://circleci.com/gh/artsy/positron); merged PRs to artsy/positron#master are automatically deployed to staging. PRs from `staging` to `release` are automatically deployed to production. [Start a deploy...](https://github.com/artsy/positron/compare/release...staging?expand=1) -- **Point Person:** [@eessex](https://github.com/eessex) +- **Point Person:** N/A [![Build Status](https://circleci.com/gh/artsy/positron/tree/master.svg?style=svg)](https://circleci.com/gh/artsy/positron/tree/master) [![codecov](https://codecov.io/gh/artsy/positron/branch/master/graph/badge.svg)](https://codecov.io/gh/artsy/positron) ## Setup -### Preparation - -- Fork Positron to your Github account in the Github UI. - -- Clone your repo locally (substitute your Github username). - -``` -git clone git@github.com:craigspaeth/positron.git && cd positron -``` -- Setup [Hokusai](https://github.com/artsy/README/blob/master/playbooks/hokusai.md#quickstart) - -- Copy `.env.example` to `.env` in the root of the project and edit all `REPLACE` values with sensitive configuration obtained from `positron-staging`. Use the following command: - -``` -hokusai staging env get | grep -E `cat .env.example | grep REPLACE | cut -f1 -d= | xargs | tr ' ' \|` | sed -e 's/:\ /=/g' | sed -e 's/ //g' -``` - -### Installs (skip if you use hokusai dev, please see section below) - -- Install [NVM](https://github.com/creationix/nvm) -- Install Node 12 - -``` -nvm install 12 -nvm alias default 12 -``` - -- Install node modules +Clone the project: ``` -yarn install +git clone git@github.com:artsy/positron.git && cd positron ``` -- Positron uses MongoDB as a database. To install MongoDB using homebrew do the following, if you would prefer to install manually check the documentation at [MongoDB](http://docs.mongodb.org/manual/tutorial/install-mongodb-on-os-x/) +Run the setup script: ``` -brew install mongodb-community +scripts/setup.sh ``` -- Start the MongoDB server +> NOTE: for nvm users, after setup finishes: nvm use -``` -mongod -``` - -- Install and run elasticsearch +Start the server: ``` -brew install elasticsearch -brew services start elasticsearch +yarn start ``` ### Prepare database @@ -97,45 +65,34 @@ With MongoDB running locally, follow these steps to create a dummy channel: } ``` -If you are using Hokusai dev, start the stack as mentioned in subsequent section, edit the database as mentioned in this step, then restart the stack. +If you are using Hokusai dev, edit the database as mentioned in this step, then restart the stack. - -### Start the server - -#### Using Yarn - -``` -yarn start -``` - -#### Using Hokusai Dev +#### Start the server using Hokusai Dev `COMMIT_HASH=$(git rev-parse --short HEAD) hokusai dev start` This starts a new Docker Compose stack that boots MongoDB, ElasticSearch and Positron. Changes made to source-code are _not_ automatically reloaded. To shut down, press `ctrl+c` or execute `hokusai dev stop`. - Positron should now be running at [http://localhost:3005/](http://localhost:3005/), open a browser and navigate to it. That will redirect you to staging, login as an Artsy administrator and it will redirect you to `http://localhost:3005` logged into Writer. If you are an Artsy Admin, you should see the default partner gallery channel (David Zwirner). If you aren't an artsy admin you'll possibly get an Unauthorized page. You need to do one more mongo operation: edit the `users` collection and set your user's `channel_ids` to `[ ObjectId("") ]`. Once that's done you should be able to see the main writer interface. -## Run tests +## Run tests ### Using Yarn +> Mongo must be running in the background for tests to work. + ``` yarn test ``` -Make sure you have mongo running in the background or most tests will not work. - ### Using Hokusai ``` hokusai test ``` - ## Debugging ### Server side @@ -148,10 +105,10 @@ yarn dev This will start the server on port `3005` with `inspect` option. -- In your Chrome go to: [Chrome Inspect](chrome://inspect) -- Under Remote Target now you should see `./index.js`, click on `inspect` link below it which will open a Chrome developer tools. +- In Chrome navigate to: [chrome://inspect](chrome://inspect) +- Under Remote Target you should see `./index.js`. Clicking on `inspect` link (below "Target") will open Chrome developer tools. -Now anywhere in your server side code you can put `debugger` and you should be able to debug. +Now anywhere in your server side code you can put `debugger` and you should be able to debug! ## Running tasks @@ -163,4 +120,4 @@ yarn task scripts/backfill.js ## Additional docs -You can find additional documentation about Positron in [doc](/doc) and [data-sync](/data-sync). +You can find additional documentation about Positron in [doc](/doc). diff --git a/hokusai/development.yml b/hokusai/development.yml index 4ccd0daed..a965ca156 100644 --- a/hokusai/development.yml +++ b/hokusai/development.yml @@ -1,13 +1,23 @@ version: "2" +volumes: + positron-mongodb-data-volume: services: positron: {% include 'templates/docker-compose-service.yml.j2' %} command: ["yarn", "dev"] - env_file: ../.env + environment: + - MONGOHQ_URL=mongodb://positron-mongodb:27017/positron + - ELASTICSEARCH_URL=http://positron-elasticsearch:9200 + # SALT is an oss value. The '$' is escaped. + - SALT=$$2a$$10$$PJrPMBadu1NPdmnshBgFbe + env_file: + - ../.env.shared + - ../.env depends_on: - positron-mongodb - positron-elasticsearch - network_mode: "host" + ports: + - 3005:3005 positron-elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16 environment: @@ -15,8 +25,13 @@ services: - xpack.security.enabled=false - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - network_mode: "host" + ports: + - 9300:9300 + - 9200:9200 positron-mongodb: image: mongo:4.0 command: ["--storageEngine=mmapv1", "--quiet", "--nojournal"] - network_mode: "host" + ports: + - 27017:27017 + volumes: + - positron-mongodb-data-volume:/data/db diff --git a/hokusai/test.yml b/hokusai/test.yml index d38678947..ff97c9c02 100644 --- a/hokusai/test.yml +++ b/hokusai/test.yml @@ -7,6 +7,8 @@ services: - CI=true - ELASTICSEARCH_URL=http://positron-elasticsearch:9200 - MONGOHQ_URL=mongodb://positron-mongodb:27017/positron-test + # SALT is an oss value. The '$' is escaped. + - SALT=$$2a$$10$$PJrPMBadu1NPdmnshBgFbe env_file: ../.env.test depends_on: - positron-mongodb diff --git a/package.json b/package.json index bbbdd0c06..ad31e15db 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "engines": { - "node": "12" + "node": "12.22.6" }, "scripts": { "assets": "sh scripts/assets.sh", @@ -21,11 +21,11 @@ "production": "sh scripts/production.sh", "publish-assets": "sh scripts/publish-assets.sh", "start": "sh scripts/start.sh", - "task": "node -r coffeescript/register -r @babel/register -r dotenv/config", + "task": "node -r coffeescript/register -r @babel/register -r ./src/lib/loadenv", "test:watch": "yarn jest -- --watch --runInBand", "test": "sh scripts/test.sh", "type-check": "tsc --pretty --noEmit", - "webpack": "node --max_old_space_size=2048 -r dotenv/config -r @babel/register node_modules/.bin/webpack --config ./webpack" + "webpack": "node --max_old_space_size=2048 -r ./src/lib/loadenv -r @babel/register node_modules/.bin/webpack --config ./webpack" }, "resolutions": { "babel-core": "7.0.0-bridge.0", diff --git a/scripts/dev.sh b/scripts/dev.sh index 00a95c756..505197f9e 100644 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -2,4 +2,4 @@ set -e -x -DEBUG=app,client,api node -r dotenv/config --inspect ./src/index.js +DEBUG=app,client,api node --inspect ./src/index.js diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 000000000..8c30e3a93 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# This assumes you have general prerequisites installed as by: +# https://github.com/artsy/potential/blob/master/scripts/setup + +# Exit if any subcommand fails +set -e + +GREEN=$(tput setaf 2) +NO_COLOR=$(tput sgr0) + +echo "Installing project dependencies from Brewfile..." +brew bundle + +if [ ! -z $NVM_DIR ]; then # skip if nvm is not available + echo "Installing Node..." + source ~/.nvm/nvm.sh + nvm install +fi + +echo "Installing dependencies..." +yarn install || (npm install --global yarn@latest && yarn install) + +echo "Download .env.shared (for common local dev config)..." +if ! aws s3 cp s3://artsy-citadel/dev/.env.positron .env.shared; then + echo "Unable to download shared config from s3. Using .env.oss!" + echo "This is expected for open source contributors." + echo "If you work at Artsy, please check your s3 access." + cp .env.oss .env.shared +fi + +if [ ! -e ".env" ]; then # skip if .env exists + echo "Initialize .env from from .env.example (for any custom configuration)..." + cat .env.example > .env +fi + +echo " +${GREEN}Setup complete! +To run the project execute: nvm use && yarn start${NO_COLOR}" diff --git a/scripts/start.sh b/scripts/start.sh index ee3ca2640..d52b4771a 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -9,7 +9,7 @@ fi export $(cat .env | grep NODE_ENV | xargs) if [ "$NODE_ENV" = "development" ]; then - node -r dotenv/config --max_old_space_size=1024 ./src + node --max_old_space_size=1024 ./src/index.js else - forever -c 'node -r dotenv/config --max_old_space_size=1024' ./src --colors + forever -c 'node --max_old_space_size=1024' ./src/index.js --colors fi diff --git a/src/index.js b/src/index.js index d8cf9ccaa..df7e5f00f 100644 --- a/src/index.js +++ b/src/index.js @@ -3,5 +3,5 @@ require("coffeescript/register") require("@babel/register")({ extensions: [".js", ".jsx", ".mjs", ".ts", ".tsx"], }) - +require("./lib/loadenv") require("./boot") diff --git a/src/lib/loadenv.js b/src/lib/loadenv.js new file mode 100644 index 000000000..f400a3067 --- /dev/null +++ b/src/lib/loadenv.js @@ -0,0 +1,33 @@ +// @ts-check + +const { readFileSync, accessSync, constants } = require("fs") +const { parse } = require("dotenv") + +function loadEnv(env) { + return checkFileExistsSync(env) ? parse(readFileSync(env)) : {} +} + +function applyToEnv(config) { + if (Object.keys(config).length !== 0) { + for (const k in config) { + if (!process.env[k]) { + process.env[k] = config[k] + } + } + } +} + +function checkFileExistsSync(filepath) { + try { + accessSync(filepath, constants.F_OK) + } catch (e) { + return false + } + return true +} + +// Load the default envs and apply to process.env +applyToEnv({ + ...loadEnv(".env.shared"), + ...loadEnv(".env"), +})