Merge pull request #1 from brian-dlee/bl-file-secrets

v0.2.1 - support file secrets

.github/workflows/node.js.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [14.x, 16.x, 18.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

README.md

@@ -18,6 +18,10 @@ files behind when the final product is uploaded.
 In an attempt to address this with my own organization and provide a stable solution for others to use, `smuggler`
 was created.
 
+This package may also help with the common issue of injecting GOOGLE_APPLICATION_CREDENTIALS (the path to 
+a Google service account credentials JSON file) or a Firebase credentials file into your application. See 
+the "The author's use case" below.
+
 ## Live demo
 
 Take a look at the live demo at https://smuggler.brian-dlee.dev. 
@@ -30,7 +34,81 @@ The code is in the [demo directory](demo).
 npm i --save @briandlee/smuggler
 ```
 
-### Create your config file
+## Config File: `.smuggler.json`
+
+The following properties are allows in the config file. 
+
+_Note: if neither `includeVariablePrefix` or `includeFiles` are supplied, there is nothing for Smuggler to do._
+
+| option                             | required | description                                               |
+|------------------------------------|----------|-----------------------------------------------------------|
+| `encryptionKeyEnvironmentVariable` | yes      | The environment variable that contains the encryption key | 
+| `encryptionIVEnvironmentVariable`  | yes      | The environment variable that contains the encryption iv  |
+| `includeVariablePrefix`            | no       | A prefix used to match environment variables to smuggle   |
+| `includeFiles`                     | no       | A list of files to smuggle (as base64)                    |
+
+### `includeFiles`
+
+Entries in `includeFiles` can take on one of two forms:
+
+**type: `file`**
+
+This is used for files that exist on disk somewhere. The file indicated by `path` will be
+read and converted to base64 before it's encrypted into the smuggler data file under the name
+indicated by `variable`. If the file does not exist or cannot be read, the operation will fail.
+
+```typescript
+interface File {
+  type: "file";
+  path: string;
+  variable: string;
+}
+```
+
+**type: `variable`**
+
+This is used for files that exist on disk somewhere, but their path is stored in an environment variable.
+The variable indicated by `name` will be read, the file it refers to will then be read, and finally it is 
+converted to base64 and encrypted into the smuggler data file under the name
+indicated by `variable`, if supplied, or `name` otherwise. If the variable is not defined, the file does not exist, or 
+the file cannot be read the operation will fail.
+
+```typescript
+interface Variable {
+  type: "variable";
+  variable?: string;
+  name: string;
+}
+```
+
+## CLI
+
+### Operation: `prepare`
+
+Use `prepare` to store environment variables and files into an encrypted file that can be uploaded as part of 
+your deployment.
+
+This is part of step-1, in a 2 phase deployment. During part 1, expose sensitive variables to the build
+environment where they can be read and exported in an encrypted format before being uploaded to the build system
+(i.e. Vercel) to be packaged with the application during the application build phase.
+
+### Operation: `generate`
+
+Use `generate` to convert prepared data from phase 1 into application files that can be bundled with your application.
+Without this step, the encoded variables do not become part of the application and will be shaken as part of builder
+optimization. If you did not prepare the data beforehand, this step can also do both the preparation and generation.
+
+It's important to run this step as part of the normal development process. Without the generated files that result
+from this step, the application will not run if smuggler is invoked. Even an empty generated file will close 
+the circuit. A common remedy is to include `smuggler generate` as part of `prestart` in your package.json.
+
+### Operation: `read`
+
+Use `read` to inspect the contents of data resulting from the `prepare` step. This is only used a debugging tool.
+
+## Following the author's use case
+
+### Creating the config file
 
 In my case, I prefix variables I want to feed into `smuggler` with `VERCEL_SECRET__`, and I store the `key` and `iv`
 parameters for encryption/decryption in the variables `VERCEL_ENCRYPTION_KEY` and `VERCEL_ENCRYPTION_IV` as recommended
@@ -45,7 +123,10 @@ and `iv` parameters to be 16 character secrets.
 {
   "encryptionKeyEnvironmentVariable": "VERCEL_ENCRYPTION_KEY",
   "encryptionIVEnvironmentVariable": "VERCEL_ENCRYPTION_IV",
-  "includeVariablePrefix": "VERCEL_SECRET__"
+  "includeVariablePrefix": "VERCEL_SECRET__",
+  "includeFiles": [
+    { "type": "variable", "name":  "GOOGLE_APPLICATION_CREDENTIALS", "variable": "GOOGLE_SERVICE_ACCOUNT_CREDENTIALS_JSON_BASE64" }
+  ]
 }
 ```
 
@@ -61,22 +142,24 @@ venturing down this avenue you've probably found you need to run custom CI calli
 
 This step creates the encrypted data in the intermediate storage location.
 
+_Note: All variables you intend to inject into your build must be available during the `prepare` phase_
+
 ```shell
 npx -y @briandlee/smuggler prepare
 ```
 
-### Add `smuggler create` to your build phase
+### Add `smuggler generate` to your build phase
 
 This step copies configuration from the intermediate storage location to the build storage location
 
 A good place for this is in `prebuild` in your `package.json`.
 
 ```json
   "scripts": {
-    "prebuild": "smuggler create",
+    "prebuild": "smuggler generate",
 ```
 
-> I also add it to `predev` to assist developers.
+> I also add it to `prestart` so the necessary files for startup with smuggler are always present.
 
 ### Load the configuration at runtime
 
@@ -88,23 +171,35 @@ secret data in memory or if your config data is really large.
 Your key and iv values must be available for read to work.
 
 ```typescript
+import { writeFileSync } from "fs";
 import { read, withDefaultReadOptions } from '@briandlee/smuggler';
+import { randomString } from "~/utils/my-random-lib"
 
-const data = read(withDefaultReadOptions({ 
-  key: process.env.VERCEL_ENCRYPTION_KEY, 
-  iv: process.env.VERCEL_ENCRYPTION_IV 
+const data = read(withDefaultReadOptions({
+  key: process.env.VERCEL_ENCRYPTION_KEY,
+  iv: process.env.VERCEL_ENCRYPTION_IV
 }));
+
+// Read and write my Google service account credentials
+if (data.GOOGLE_SERVICE_ACCOUNT_CREDENTIALS_JSON_BASE64) {
+  const filePath = `/tmp/${randomString(64)}.json`;
+  writeFileSync(
+    filePath,
+    Buffer.from(data.GOOGLE_SERVICE_ACCOUNT_CREDENTIALS_JSON_BASE64, 'base64')
+  );
+  process.env.GOOGLE_APPLICATION_CREDENTIALS = filePath;
+}
 ```
 
-### The author's use case
+### An Example
 
 See the [example](example/prisma-app) for an illustration for how the package is designed to be used.
 
 ## Caveats
 
-- At the time of writing this, I'm using Remix. If you are using another framework that uses a different build system 
-  you encounter a situation where the smuggler data is pruned from the final build (maybe in the case of Next.js and 
-  their use of [nft](https://github.com/vercel/nft)).
+ - At the time of writing this, I'm using Remix (which uses esbuild). If you are using another framework that uses a 
+   different build system you encounter a situation where the smuggler data is pruned from the final build (maybe in the 
+   case of Next.js and their use of [nft](https://github.com/vercel/nft)).
 
 ## TODO
 

demo/.smuggler.json

@@ -1,6 +1,11 @@
 {
     "encryptionKeyEnvironmentVariable": "SMUGGLER_ENCRYPTION_KEY",
     "encryptionIVEnvironmentVariable": "SMUGGLER_ENCRYPTION_IV",
-    "includeVariablePrefix": "SMUGGLER__"
+    "includeVariablePrefix": "SMUGGLER__",
+    "includeFiles": [
+      {
+        "type": "file", "path": ".smuggler/a-fake-credentials-file.json", "variable": "CREDENTIALS_JSON_BASE64"
+      }
+    ]
   }
 

demo/.smuggler/a-fake-credentials-file.json

@@ -0,0 +1,45 @@
+{
+  "_id": "630138266b891f0be50f6402",
+  "index": 0,
+  "guid": "d36bd609-4cde-4478-97fa-3af616e9e14e",
+  "isActive": true,
+  "balance": "$2,921.79",
+  "picture": "http://placehold.it/32x32",
+  "age": 38,
+  "eyeColor": "brown",
+  "name": "Amparo Mclaughlin",
+  "gender": "female",
+  "company": "ZOLAR",
+  "email": "[email protected]",
+  "phone": "+1 (908) 429-2706",
+  "address": "987 Albemarle Road, Fidelis, Washington, 5328",
+  "about": "Excepteur ea do adipisicing ut proident aliquip voluptate nisi non ullamco minim do sit ea. Aliquip reprehenderit occaecat commodo in pariatur ex dolor sint aliqua tempor ipsum dolore mollit dolore. Ad tempor quis dolore irure esse ullamco incididunt anim et officia sint irure adipisicing. Adipisicing dolore cupidatat voluptate non mollit irure consectetur pariatur excepteur. Culpa adipisicing excepteur anim occaecat id dolor cupidatat consectetur enim culpa magna do culpa occaecat.\r\n",
+  "registered": "2019-08-02T12:02:58 +07:00",
+  "latitude": 87.967907,
+  "longitude": 157.518182,
+  "tags": [
+    "pariatur",
+    "occaecat",
+    "aute",
+    "sunt",
+    "incididunt",
+    "minim",
+    "nulla"
+  ],
+  "friends": [
+    {
+      "id": 0,
+      "name": "Peterson Grimes"
+    },
+    {
+      "id": 1,
+      "name": "Dotson Burke"
+    },
+    {
+      "id": 2,
+      "name": "Carson Harrell"
+    }
+  ],
+  "greeting": "Hello, Amparo Mclaughlin! You have 9 unread messages.",
+  "favoriteFruit": "strawberry"
+}

demo/.vercelignore

@@ -0,0 +1 @@
+.env

demo/app/entry.server.tsx

@@ -1,28 +1,39 @@
-import type { EntryContext } from "@remix-run/node";
-import { RemixServer } from "@remix-run/react";
-import invariant from "invariant";
-import { renderToString } from "react-dom/server";
-import { initialize } from "./lib/config.server"
+import type { EntryContext } from '@remix-run/node';
+import { RemixServer } from '@remix-run/react';
+import invariant from 'invariant';
+import { renderToString } from 'react-dom/server';
+import { initialize } from './lib/config.server';
 
-initialize()
+initialize();
 
 // While it's a little ridiculous to load an image synchronously during startup...
-invariant(process.env.FILE_BASE64, 'The super secret and necessary file is missing')
-console.log(`Loaded secret image. Size=${Buffer.byteLength(Buffer.from(process.env.FILE_BASE64, 'base64')) / 1000}kb`)
+invariant(process.env.FILE_BASE64, 'The super secret and necessary file is missing');
+console.log(
+  `Loaded secret image. Size=${
+    Buffer.byteLength(Buffer.from(process.env.FILE_BASE64, 'base64')) / 1000
+  }kb`
+);
+
+invariant(process.env.CREDENTIALS_JSON_BASE64, 'The fake credentials file is missing');
+console.log(
+  `Loaded credentials file. (Displaying first 128 chracters) ${JSON.stringify(
+    JSON.parse(Buffer.from(process.env.CREDENTIALS_JSON_BASE64, 'base64').toString('utf-8')),
+    null,
+    2
+  ).slice(0, 128)}...`
+);
 
 export default function handleRequest(
   request: Request,
   responseStatusCode: number,
   responseHeaders: Headers,
   remixContext: EntryContext
 ) {
-  let markup = renderToString(
-    <RemixServer context={remixContext} url={request.url} />
-  );
+  let markup = renderToString(<RemixServer context={remixContext} url={request.url} />);
 
-  responseHeaders.set("Content-Type", "text/html");
+  responseHeaders.set('Content-Type', 'text/html');
 
-  return new Response("<!DOCTYPE html>" + markup, {
+  return new Response('<!DOCTYPE html>' + markup, {
     status: responseStatusCode,
     headers: responseHeaders,
   });

demo/app/routes/index.tsx

@@ -3,6 +3,7 @@ import { json } from '@remix-run/node';
 import { useLoaderData } from '@remix-run/react';
 import { load } from '~/lib/config.server';
 import mainStyleSheetUrl from '~/styles/main.css';
+import { btoa } from 'buffer';
 
 interface LoaderData {
   config: Record<string, string>;
@@ -34,7 +35,8 @@ export const loader: LoaderFunction = async () => {
           'SMUGGLER_ENCRYPTION_KEY',
           'SMUGGLER_ENCRYPTION_IV',
           'API_KEY',
-          'FILE_BASE64'
+          'FILE_BASE64',
+          'CREDENTIALS_JSON_BASE64'
         ),
       },
       200
@@ -58,8 +60,17 @@ export default function Index() {
         limit. It doesn't discriminate.
       </p>
       {config.FILE_BASE64 && (
-        <div className="image-container">
-          <img alt="it's been smuggled" src={`data:image/webp;base64,${config.FILE_BASE64}`} />
+        <div>
+          <h3>Exhibit 1: The smuggled image file</h3>
+          <div className="image-container">
+            <img alt="it's been smuggled" src={`data:image/webp;base64,${config.FILE_BASE64}`} />
+          </div>
+        </div>
+      )}
+      {config.CREDENTIALS_JSON_BASE64 && (
+        <div>
+          <h3>Exhibit 2: The smuggled credentials JSON</h3>
+          <pre>{JSON.stringify(JSON.parse(atob(config.CREDENTIALS_JSON_BASE64)), null, 2)}</pre>
         </div>
       )}
       <p>

demo/package.json

@@ -9,7 +9,7 @@
     "start": "remix-serve build"
   },
   "dependencies": {
-    "@briandlee/smuggler": "0.2.0-dev2",
+    "@briandlee/smuggler": "^0.2.1-dev.3",
     "@remix-run/node": "^1.6.6",
     "@remix-run/react": "^1.6.6",
     "@remix-run/serve": "^1.6.6",