feat: turn-key env (#2972)

* turn-key setup inital pass

- add setup script installing all requirements
- add brewfile to install mongo and elasticsearch
- edit env.example removing shared configs
- update development compose file
- pin node version 12.22.6
- update Dockerfile to use node 12.22

* implement loadenv.js

* update README

* update development file

* escape SALT '$' char for mac dockerized dev to work

* introduce .env.oss

* proload loadenv instead of dotenv for task and webpack scripts

* minor edits

* edit docker-compose files

* switch setup script to execute in bash, use tput for colors

* fix a broken link in README

* add a note about how to use .env

.dockerignore

@@ -20,4 +20,5 @@ tmp
 node_modules
 npm-debug.log
 .vscode
-.circleci
+.circleci
+Brewfile*

.env.example

@@ -1,16 +1,5 @@
-NODE_ENV=development
-PORT=3005
-APP_NAME=positron-staging
-APP_URL=http://localhost:3005
-API_URL=http://localhost:3005/api
-FORCE_URL=https://staging.artsy.net
-ARTSY_URL=https://stagingapi.artsy.net
-SESSION_SECRET=p0s1tr0n
-ELASTICSEARCH_PORT=9200
-
-# Local development only for the below; refers to the key in docker-compose.yml
-ELASTICSEARCH_URL=http://localhost:9200
-MONGOHQ_URL=mongodb://localhost:27017/positron
+# The .env.example file provides an *initial* .env file.
+# Local overrides should go in the .env file.
 
 ##
 # Set the DATADOG_AGENT_HOSTNAME to some value and start a local agent via Docker
@@ -37,40 +26,3 @@ MONGOHQ_URL=mongodb://localhost:27017/positron
 #```
 ##
 DATADOG_AGENT_HOSTNAME=datadog-agent
-
-# Needed to link Reaction: `yarn link @artsy/reaction`
-GRAPHQL_ENDPOINT=https://metaphysics-staging.artsy.net
-
-TECH_SUPPORT=[email protected]
-DEBUG=api,client,app
-SALT=$2a$10$PJrPMBadu1NPdmnshBgFbe
-API_MAX=100
-API_PAGE_SIZE=10
-GEMINI_APP=writer-staging
-FORCE_COLOR=1
-WEBFONT_URL=http://webfonts.artsy.net/
-IP_BLACKLIST=
-NO_INDEX_CHANNELS=
-DEV_FEATURES=1
-
-# OSS version of the .env file
-# Note: The ARTSY_ID & ARTSY_SECRET are known keys for Artsy OSS
-# projects, and are not a problem. OSS people: Please don't abuse the keys,
-# as then we'll have to change it, making it harder for others to learn from.
-# As such, these keys do not come under the Artsy security bounty.
-
-ARTSY_ID=e750db60ac506978fc70
-ARTSY_SECRET=3a33d2085cbd1176153f99781bbce7c6
-EDITORIAL_CHANNEL=REPLACE_ME
-SEGMENT_WRITE_KEY=REPLACE_ME
-GRAVITY_CLOUDFRONT_URL=REPLACE_ME
-FULCRUM_BUCKET=REPLACE_ME
-S3_KEY=REPLACE_ME
-S3_SECRET=REPLACE_ME
-SAILTHRU_KEY=REPLACE_ME
-SAILTHRU_SECRET=REPLACE_ME
-SECURE_IMAGES_URL=REPLACE_ME
-FB_PAGE_ID=REPLACE_ME
-DEFAULT_PARTNER_ID=REPLACE_ME
-SENTRY_PUBLIC_DSN=REPLACE_ME
-SENTRY_PRIVATE_DSN=REPLACE_ME

.env.oss

@@ -0,0 +1,30 @@
+# OSS version of the .env.shared file
+# Note: The ARTSY_ID & ARTSY_SECRET are known keys for Artsy OSS
+# projects, and are not a problem. OSS people: Please don't abuse the keys,
+# as then we'll have to change it, making it harder for others to learn from.
+# As such, these keys do not come under the Artsy security bounty.
+
+ARTSY_ID=e750db60ac506978fc70
+ARTSY_SECRET=3a33d2085cbd1176153f99781bbce7c6
+NODE_ENV=development
+PORT=3005
+APP_NAME=positron-staging
+APP_URL=http://localhost:3005
+API_URL=http://localhost:3005/api
+FORCE_URL=https://staging.artsy.net
+ARTSY_URL=https://stagingapi.artsy.net
+SESSION_SECRET=p0s1tr0n
+ELASTICSEARCH_PORT=9200
+ELASTICSEARCH_URL=http://localhost:9200
+MONGOHQ_URL=mongodb://localhost:27017/positron
+GRAPHQL_ENDPOINT=https://metaphysics-staging.artsy.net # Needed to link Reaction: `yarn link @artsy/reaction`
+DEBUG=api,client,app
+SALT=$2a$10$PJrPMBadu1NPdmnshBgFbe
+API_MAX=100
+API_PAGE_SIZE=10
+GEMINI_APP=writer-staging
+FORCE_COLOR=1
+WEBFONT_URL=http://webfonts.artsy.net/
+IP_BLACKLIST=
+NO_INDEX_CHANNELS=
+DEV_FEATURES=1

.gitignore

@@ -13,8 +13,7 @@ npm-debug.log
 .DS_Store
 dump.rdb
 .DS_Store
-.env
-.env.ignore
+.env*
 .cache
 
 data
@@ -28,3 +27,9 @@ coverage*
 .nyc_output/
 
 cypress.env.json
+Brewfile.lock.json
+
+# Do not ignore
+!.env.example
+!.env.test
+!.env.oss

.nvmrc

@@ -1 +1 @@
-12
+12.22.6

Brewfile

@@ -0,0 +1,4 @@
+tap 'artsy/formulas'
+tap 'mongodb/brew'
+brew '[email protected]', link: true, restart_service: true
+brew '[email protected]', restart_service: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12.18-alpine
+FROM node:12.22-alpine
 
 ENV PORT 3005
 EXPOSE 3005

README.md

@@ -8,7 +8,7 @@
 
 - **State:** production
 - **Production:** [https://writer.artsy.net/](https://writer.artsy.net/) | [Kubernetes](https://kubernetes.prd.artsy.systems/#!/deployment/default/positron-web?namespace=default)
-- **Staging:** [http://stagingwriter.artsy.net/](http://stagingwriter.artsy.net//) | [Kubernetes](https://kubernetes.stg.artsy.systems/#!/deployment/default/positron-web?namespace=default)
+- **Staging:** [https://stagingwriter.artsy.net/](https://stagingwriter.artsy.net/) | [Kubernetes](https://kubernetes.stg.artsy.systems/#!/deployment/default/positron-web?namespace=default)
 - **Logs:**
   - [Production](https://papertrailapp.com/groups/3675843/events?q=host%3Apositron-web)
   - [Staging](https://papertrailapp.com/groups/3674473/events?q=host%3Apositron-web)
@@ -18,62 +18,30 @@
 - **MongoDB:** [Atlas](https://cloud.mongodb.com/v2/5be44a7aff7a254a8327cd3a#clusters)
 - **Github:** [https://github.com/artsy/positron/](https://github.com/artsy/positron/)
 - **CI:** [CircleCI](https://circleci.com/gh/artsy/positron); merged PRs to artsy/positron#master are automatically deployed to staging. PRs from `staging` to `release` are automatically deployed to production. [Start a deploy...](https://github.com/artsy/positron/compare/release...staging?expand=1)
-- **Point Person:** [@eessex](https://github.com/eessex)
+- **Point Person:** N/A
 
 [![Build Status](https://circleci.com/gh/artsy/positron/tree/master.svg?style=svg)](https://circleci.com/gh/artsy/positron/tree/master) [![codecov](https://codecov.io/gh/artsy/positron/branch/master/graph/badge.svg)](https://codecov.io/gh/artsy/positron)
 
 ## Setup
 
-### Preparation
-
-- Fork Positron to your Github account in the Github UI.
-
-- Clone your repo locally (substitute your Github username).
-
-```
-git clone [email protected]:craigspaeth/positron.git && cd positron
-```
-- Setup [Hokusai](https://github.com/artsy/README/blob/master/playbooks/hokusai.md#quickstart)
-
-- Copy `.env.example` to `.env` in the root of the project and edit all `REPLACE` values with sensitive configuration obtained from `positron-staging`. Use the following command:
-
-```
-hokusai staging env get | grep -E `cat .env.example | grep REPLACE | cut -f1 -d= | xargs | tr ' ' \|` | sed -e 's/:\ /=/g' | sed -e 's/ //g'
-```
-
-### Installs (skip if you use hokusai dev, please see section below)
-
-- Install [NVM](https://github.com/creationix/nvm)
-- Install Node 12
-
-```
-nvm install 12
-nvm alias default 12
-```
-
-- Install node modules
+Clone the project:
 
 ```
-yarn install
+git clone [email protected]:artsy/positron.git && cd positron
 ```
 
-- Positron uses MongoDB as a database. To install MongoDB using homebrew do the following, if you would prefer to install manually check the documentation at [MongoDB](http://docs.mongodb.org/manual/tutorial/install-mongodb-on-os-x/)
+Run the setup script:
 
 ```
-brew install mongodb-community
+scripts/setup.sh
 ```
 
-- Start the MongoDB server
+> NOTE: for nvm users, after setup finishes: nvm use
 
-```
-mongod
-```
-
-- Install and run elasticsearch
+Start the server:
 
 ```
-brew install elasticsearch
-brew services start elasticsearch
+yarn start
 ```
 
 ### Prepare database
@@ -97,45 +65,34 @@ With MongoDB running locally, follow these steps to create a dummy channel:
 }
 ```
 
-If you are using Hokusai dev, start the stack as mentioned in subsequent section, edit the database as mentioned in this step, then restart the stack.
+If you are using Hokusai dev, edit the database as mentioned in this step, then restart the stack.
 
-
-### Start the server
-
-#### Using Yarn
-
-```
-yarn start
-```
-
-#### Using Hokusai Dev
+#### Start the server using Hokusai Dev
 
 `COMMIT_HASH=$(git rev-parse --short HEAD) hokusai dev start`
 
 This starts a new Docker Compose stack that boots MongoDB, ElasticSearch and Positron. Changes made to source-code are _not_ automatically reloaded. To shut down, press `ctrl+c` or execute `hokusai dev stop`.
 
-
 Positron should now be running at [http://localhost:3005/](http://localhost:3005/), open a browser and navigate to it. That will redirect you to staging, login as an Artsy administrator and it will redirect you to `http://localhost:3005` logged into Writer.
 
 If you are an Artsy Admin, you should see the default partner gallery channel (David Zwirner). If you aren't an artsy admin you'll possibly get an Unauthorized page. You need to do one more mongo operation: edit the `users` collection and set your user's `channel_ids` to `[ ObjectId("<your_above_channel_id>") ]`. Once that's done you should be able to see the main writer interface.
 
-##  Run tests
+## Run tests
 
 ### Using Yarn
 
+> Mongo must be running in the background for tests to work.
+
 ```
 yarn test
 ```
 
-Make sure you have mongo running in the background or most tests will not work.
-
 ### Using Hokusai
 
 ```
 hokusai test
 ```
 
-
 ## Debugging
 
 ### Server side
@@ -148,10 +105,10 @@ yarn dev
 
 This will start the server on port `3005` with `inspect` option.
 
-- In your Chrome go to: [Chrome Inspect](chrome://inspect)
-- Under Remote Target now you should see `./index.js`, click on `inspect` link below it which will open a Chrome developer tools.
+- In Chrome navigate to: [chrome://inspect](chrome://inspect)
+- Under Remote Target you should see `./index.js`. Clicking on `inspect` link (below "Target") will open Chrome developer tools.
 
-Now anywhere in your server side code you can put `debugger` and you should be able to debug.
+Now anywhere in your server side code you can put `debugger` and you should be able to debug!
 
 ## Running tasks
 
@@ -163,4 +120,4 @@ yarn task scripts/backfill.js
 
 ## Additional docs
 
-You can find additional documentation about Positron in [doc](/doc) and [data-sync](/data-sync).
+You can find additional documentation about Positron in [doc](/doc).

hokusai/development.yml

@@ -1,22 +1,37 @@
 version: "2"
+volumes:
+  positron-mongodb-data-volume:
 services:
   positron:
 {% include 'templates/docker-compose-service.yml.j2' %}
     command: ["yarn", "dev"]
-    env_file: ../.env
+    environment:
+      - MONGOHQ_URL=mongodb://positron-mongodb:27017/positron
+      - ELASTICSEARCH_URL=http://positron-elasticsearch:9200
+      # SALT is an oss value. The '$' is escaped.
+      - SALT=$$2a$$10$$PJrPMBadu1NPdmnshBgFbe
+    env_file:
+      - ../.env.shared
+      - ../.env
     depends_on:
       - positron-mongodb
       - positron-elasticsearch
-    network_mode: "host"
+    ports:
+      - 3005:3005
   positron-elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16
     environment:
       - discovery.type=single-node
       - xpack.security.enabled=false
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    network_mode: "host"
+    ports:
+      - 9300:9300
+      - 9200:9200
   positron-mongodb:
     image: mongo:4.0
     command: ["--storageEngine=mmapv1", "--quiet", "--nojournal"]
-    network_mode: "host"
+    ports:
+      - 27017:27017
+    volumes:
+      - positron-mongodb-data-volume:/data/db

hokusai/test.yml

@@ -7,6 +7,8 @@ services:
       - CI=true
       - ELASTICSEARCH_URL=http://positron-elasticsearch:9200
       - MONGOHQ_URL=mongodb://positron-mongodb:27017/positron-test
+      # SALT is an oss value. The '$' is escaped.
+      - SALT=$$2a$$10$$PJrPMBadu1NPdmnshBgFbe
     env_file: ../.env.test
     depends_on:
       - positron-mongodb

package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "engines": {
-    "node": "12"
+    "node": "12.22.6"
   },
   "scripts": {
     "assets": "sh scripts/assets.sh",
@@ -21,11 +21,11 @@
     "production": "sh scripts/production.sh",
     "publish-assets": "sh scripts/publish-assets.sh",
     "start": "sh scripts/start.sh",
-    "task": "node -r coffeescript/register -r @babel/register -r dotenv/config",
+    "task": "node -r coffeescript/register -r @babel/register -r ./src/lib/loadenv",
     "test:watch": "yarn jest -- --watch --runInBand",
     "test": "sh scripts/test.sh",
     "type-check": "tsc --pretty --noEmit",
-    "webpack": "node --max_old_space_size=2048 -r dotenv/config -r @babel/register node_modules/.bin/webpack --config ./webpack"
+    "webpack": "node --max_old_space_size=2048 -r ./src/lib/loadenv -r @babel/register node_modules/.bin/webpack --config ./webpack"
   },
   "resolutions": {
     "babel-core": "7.0.0-bridge.0",

scripts/dev.sh

@@ -2,4 +2,4 @@
 
 set -e -x
 
-DEBUG=app,client,api node -r dotenv/config --inspect ./src/index.js
+DEBUG=app,client,api node --inspect ./src/index.js