Merge pull request #2 from Totodore/develop

Develop

Readme.md

@@ -1 +1,105 @@
-# Docker-CI/CD
+# Docker-CI
+
+Docker-CI is a little program which allow you to implement easy continuous integration through Github Workflows and docker-compose. It uses labels to set the different options to enable Docker-ci for each container.
+
+Docker-CI watch for container creations, it means that you don't have to restart Docker-CI whenever you update a container configuration.
+
+Docker-CI will then create a route corresponding to this pattern : ```http(s)://0.0.0.0[:port]/deploy/:appName``` where the appName correspond to the name you gave to your container or to the name you gave through the option ```docker-ci.name```
+You can then set a Github Automation with an [Image building](https://github.com/actions/starter-workflows/blob/a571f2981ab5a22dfd9158f20646c2358db3654c/ci/docker-publish.yml) and you can then add a webhook to trigger the above url when the image is built and stored in the Github Package Registry
+## Example
+
+### docker-compose.yml of docker-ci app
+```yaml
+version: "3"
+services:
+  docker-ci:
+    container_name: totodore/docker-ci:latest
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: always
+    ports:
+      - "5050:80"
+    environment:
+      - PORT=80
+      - NODE_ENV=production
+```
+### docker-compose.yml of application Docker-CI (example of App with a Continuous integration workflow) :
+```yaml
+version: "3.7"
+services:
+  app:
+    image: ghcr.io/totodore/automate:latest  ##The github registry link
+    container_name: automate
+    tty: true
+    expose:
+      - 80
+    restart: always
+    labels:
+      - "docker-ci.enabled=true"
+      - "docker-ci.name=automate"
+```
+
+### docker-publish in the github repo :
+```yaml
+name: Docker
+
+on:
+  push:
+    # Publish `master` as Docker `latest` image.
+    branches:
+      - master
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+
+env:
+  # TODO: Change variable to your image's name.
+  IMAGE_NAME: automate
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build image
+        run: docker build . --file Dockerfile --tag $IMAGE_NAME
+
+      - name: Log into GitHub Container Registry
+      # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT`
+        run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Push image to GitHub Container Registry
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "master" ] && VERSION=latest
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
+  deploy:
+    needs: push
+    name: deploy
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Deploy docker container webhook
+        uses: joelwmale/webhook-action@master
+        env:
+          WEBHOOK_URL: ${{ secrets.DEPLOY_WEBHOOK_URL }} #This Docker secret correspond to http(s)://IP[:port]/deploy/automate
+```
+
+## Labels available :
+|Name|Type|Description|
+|----|----|-----------|
+| ```docker-ci.enable```|```boolean```|Enable CI for this container, an endpoint will be created for this container and whenever it will be called the container image will be repulled and the container will be recreated (total update of the container)|
+| ```docker-ci.name```|```string (Optional)```|Set a custom name for the endpoint, by default it is the name of the container|

package.json

@@ -5,7 +5,7 @@
   "main": "bin/index.js",
   "scripts": {
     "build": "tsc",
-    "start": "node ./bin/app.js",
+    "start": "node ./bin/index.js",
     "dev": "tsc-watch --onSuccess 'npm start' --onFailure 'echo Error during compilation'"
   },
   "repository": {
@@ -24,10 +24,14 @@
   },
   "homepage": "https://github.com/Totodore/github-ci#readme",
   "dependencies": {
-    "dockerode": "^3.2.1"
+    "dockerode": "^3.2.1",
+    "dotenv": "^8.2.0",
+    "express": "^4.17.1"
   },
   "devDependencies": {
     "@types/dockerode": "^3.2.1",
+    "@types/dotenv": "^8.2.0",
+    "@types/express": "^4.17.9",
     "tsc-watch": "^4.2.9",
     "typescript": "^4.1.2"
   }

src/docker.ts

@@ -0,0 +1,88 @@
+import { DockerCiLabels } from './models/docker-ci-labels.model';
+import { DockerEventsModel } from './models/docker-events.model';
+import * as Docker from "dockerode";
+import { Logger } from "./utils/logger";
+
+export class DockerManager {
+  private _docker: Docker;
+  private _logger = new Logger(this);
+
+  public async init() {
+    try {
+      this._docker = new Docker({ socketPath: "/var/run/docker.sock" })
+      await this._docker.ping();
+    } catch (e) {
+      this._logger.error("Error connecting to Docker.", e);
+      process.exit(1);
+    }
+  }
+
+  /**
+   * Add a event listener on container events
+   * @param e the event to listen
+   * @param callback 
+   */
+  public async addContainerEventListener(e: (keyof typeof DockerEventsModel.ContainerEvents), callback: (res: DockerEventsModel.EventResponse) => void) {
+    (await this._docker.getEvents()).on("data", (rawData) => {
+      const data: DockerEventsModel.EventResponse = JSON.parse(rawData);
+      if (data.Type === "container" && data.Action === e) {
+        callback(data);
+      }
+    });
+  }
+
+  public getContainer(id: string): Docker.Container {
+    return this._docker.getContainer(id);
+  }
+  public getImage(name: string): Docker.Image {
+    return this._docker.getImage(name);
+  }
+
+  /**
+   * Get all docker container enableb with docker-ci.enabled
+   * Return a object where key is the id of the container and value
+   * Is the name of the webhook or the name of the container
+   */
+  public async getAllContainersEnabled(): Promise<{ [k: string]: string }> {
+    const containers = (await this._docker.listContainers());
+    const response: { [k: string]: string } = {};
+    for (const container of containers) {
+      const labels: DockerCiLabels = container.Labels;
+      if (labels["docker-ci.enable"] === "true")
+        response[container.Id] = labels["docker-ci.name"] || (await this.getContainer(container.Id).inspect()).Name;
+    }
+    return response;
+  }
+
+  /**
+   * Pull an image from its tag
+   * @returns true in case of success
+   */
+  public async pullImage(imageName: string): Promise<boolean> {
+    try {
+      const imageInfos = await this.getImage(imageName).inspect();
+      this._logger.log("Pulling : ", ...imageInfos.RepoTags);
+      for(const tag of imageInfos.RepoTags)
+        await this._docker.pull(tag);
+      return true; 
+    } catch (e) {
+      this._logger.error("Error pulling image", e);
+      return false;
+    }
+  }
+
+  /**
+   * Recreate a container from its ID
+   * @param containerId 
+   */
+  public async recreateContainer(containerId: string) {
+    const container = this.getContainer(containerId);
+    const infos = await container.inspect();
+    this._logger.log("Stopping container");
+    await container.stop();
+    this._logger.log("Removing container");
+    await container.remove();
+    this._logger.log("Recreating container");
+    (await this._docker.createContainer(infos.Config)).start();
+  }
+}

src/index.ts

@@ -1,24 +1,92 @@
+import { WebhooksManager } from './webhooks';
+import { DockerManager } from './docker';
+import { DockerCiLabels } from './models/docker-ci-labels.model';
+import { DockerEventsModel } from './models/docker-events.model';
 import { Logger } from './utils/logger';
-import * as Docker from "dockerode";
 
 class App {
 
-  private _docker: Docker;
-  private _logger = new Logger(this);
+  private readonly _logger = new Logger(this);
+  private _dockerManager = new DockerManager();
+  private _webhooksManager = new WebhooksManager();
 
   public async init() {
-    this._connect();
-    console.log(await this._docker.getEvents());
+    this._logger.log("Connecting to docker endpoint");
+    await this._dockerManager.init();
+    await this._webhooksManager.init();
+
+    this._dockerManager.addContainerEventListener("start", (res) => this._onCreateContainer(res));
+
+    this._logger.log("Connected to docker endpoint.");
+    this._logger.log("Watching container creation.");
+    this._logger.log(`Listening webhooks on ${this._webhooksManager.webhookUrl}/:id`);
+    this._logger.log("Loading configuration from existing containers");
+    this.loadContainerConf();
+  }
+
+  /**
+   * Load configuration from the existing containers
+   * with the labels docker-ci.enabled=true
+   */
+  public async loadContainerConf() {
+    const containers = await this._dockerManager.getAllContainersEnabled();
+    this._logger.log(Object.values(containers).length, "containers with webhooks detected");
+    for (const containerId in containers) {
+      this._logger.log("Adding route for container named", containers[containerId]);
+      this._addContainerConf(containers[containerId], containerId);
+    }
+  }
+
+  /**
+   * Called when a container is recreated/created
+   * If docker-ci.enable is true :
+   * Add a route to the webhooks with the id of the container or the given name "docker-ci.name"
+   */
+  private async _onCreateContainer(res: DockerEventsModel.EventResponse) {
+    const containerName = res.Actor.Attributes.name;
+    this._logger.log("Container creation detected :", containerName);
+
+    try {
+      const containerInfos = await this._dockerManager.getContainer(res.Actor.ID).inspect();
+      const labels: DockerCiLabels = containerInfos.Config.Labels;
+      const routeId = labels["docker-ci.name"] || containerName;
+      if (labels["docker-ci.enable"] === "true") {
+        this._logger.log("Docker-ci enabled, adding container to webhook conf");
+        this._addContainerConf(routeId, containerInfos.Id);
+      }
+      else
+        this._logger.log("Docker-ci not enabled, skipping...");
+
+    } catch (e) {
+      this._logger.error("Error with getting informations of container :", e);
+    }
   }
 
-  private _connect() {
+  /**
+   * Add the route to wenhooks
+   * @param routeId 
+   * @param id 
+   */
+  private async _addContainerConf(routeId: string, id: string) {
+    this._logger.log(`New webhook available at : ${this._webhooksManager.webhookUrl}/${routeId}`);
+    this._webhooksManager.addRoute(routeId, () => this._onUrlTriggered(id));
+  }
+
+  /**
+   * Triggered when someone call the url 
+   * @param id the id/name of the container to reload
+   */
+  private async _onUrlTriggered(id: string) {
     try {
-      this._docker = new Docker({ socketPath: "/var/run/docker.sock" })
+      const containerInfos = await this._dockerManager.getContainer(id).inspect();
+      if (!await this._dockerManager.pullImage(containerInfos.Image))
+        throw "Error Pulling Image";
+      await this._dockerManager.recreateContainer(id);
     } catch (e) {
-      this._logger.error("Error connecting to Docker", e);
-      process.exit(1);
+      this._logger.error("Error Pulling Image and Recreating Container", e);
     }
   }
+
 }
 
 new App().init();

src/models/docker-ci-labels.model.ts

@@ -0,0 +1,6 @@
+export interface DockerCiLabels {
+  "docker-ci.enable"?: string;
+  "docker-ci.name"?: string;
+  "docker-ci.secret"?: string;
+  "docker-ci.url"?: string;
+}

src/models/docker-events.model.ts

@@ -0,0 +1,52 @@
+import { Image } from "dockerode";
+
+export namespace DockerEventsModel {
+  export enum ContainerEvents {
+    "attach",
+    "commit",
+    "copy",
+    "create",
+    "destroy",
+    "detach",
+    "die",
+    "exec_create",
+    "exec_detach",
+    "exec_die",
+    "exec_start",
+    "export",
+    "health_status",
+    "kill",
+    "oom",
+    "pause",
+    "rename",
+    "resize",
+    "restart",
+    "start",
+    "stop",
+    "top",
+    "unpause",
+    "update",
+  };
+  export enum ImageEvents {
+    "delete",
+    "import",
+    "load",
+    "pull",
+    "push",
+    "save",
+    "tag",
+    "untag",
+  }
+
+  export interface EventResponse {
+    Type: "container" | "image",
+    Action: keyof typeof ContainerEvents | keyof typeof ImageEvents,
+    Actor: {
+      ID: string;
+      Attributes: { [k: string]: string }
+    },
+    Time: number,
+    TimeNano: number
+  }
+
+}

src/models/docker-images.model.ts

@@ -0,0 +1,10 @@
+export namespace DockerImagesModel {
+  export interface PullImageParameters {
+    fromImage?: string;
+    fromSrc?: string;
+    repo?: string;
+    tag?: string;
+    platform?: string;
+    "X-Registry-Auth"?: string;
+  }
+}