diff --git a/.castor/docker.php b/.castor/docker.php index d19b336..3e38709 100644 --- a/.castor/docker.php +++ b/.castor/docker.php @@ -23,6 +23,7 @@ use function Castor\open; use function Castor\run; use function Castor\variable; +use function Castor\yaml_parse; #[AsTask(description: 'Displays some help and available urls for the current project', namespace: '')] function about(): void @@ -84,7 +85,6 @@ function build( $command = [ ...$command, 'build', - '--build-arg', 'USER_ID=' . variable('user_id'), '--build-arg', 'PHP_VERSION=' . variable('php_version'), '--build-arg', 'PROJECT_NAME=' . variable('project_name'), ]; @@ -295,6 +295,107 @@ function workers_stop(): void stop(profiles: ['worker']); } +#[AsTask(description: 'Push images cache to the registry', namespace: 'docker', name: 'push', aliases: ['push'])] +function push(): void +{ + // Generate bake file + $composeFile = variable('docker_compose_files'); + $composeFile[] = 'docker-compose.builder.yml'; + + $targets = []; + + /** @var string|null $registry */ + $registry = variable('registry'); + + if ($registry === null || $registry === '') { + throw new \RuntimeException('You must define a registry to push images.'); + } + + foreach ($composeFile as $file) { + $path = variable('root_dir') . '/infrastructure/docker/' . $file; + $content = file_get_contents($path); + $data = yaml_parse($content); + + foreach ($data['services'] ?? [] as $service => $config) { + $cacheFrom = $config['build']['cache_from'][0] ?? null; + + if (null === $cacheFrom) { + continue; + } + + $cacheFrom = explode(',', $cacheFrom); + $reference = null; + $type = null; + + if (count($cacheFrom) === 1) { + $reference = $cacheFrom[0]; + $type = 'registry'; + } else { + foreach ($cacheFrom as $part) { + $from = explode('=', $part); + + if (count($from) !== 2) { + continue; + } + + if ($from[0] === 'type') { + $type = $from[1]; + } + + if ($from[0] === 'ref') { + $reference = $from[1]; + } + } + } + + $targets[] = [ + 'reference' => $reference, + 'type' => $type, + 'context' => $config['build']['context'], + 'dockerfile' => $config['build']['dockerfile'] ?? 'Dockerfile', + 'target' => $config['build']['target'] ?? null, + ]; + } + } + + $content = sprintf(<< sprintf('"%s"', $target['target']), $targets))); + + + foreach ($targets as $target) { + $reference = str_replace('${REGISTRY:-}', $registry, $target['reference'] ?? ''); + + $content .= sprintf(<< variable('user_id'), 'COMPOSER_CACHE_DIR' => variable('composer_cache_dir'), 'PHP_VERSION' => variable('php_version'), + 'REGISTRY' => variable('registry'), ]) ; diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml new file mode 100644 index 0000000..1c867e9 --- /dev/null +++ b/.github/workflows/cache.yml @@ -0,0 +1,36 @@ +name: Push docker image to registry + +"on": + push: + # Only run this job when pushing to the main branch + branches: ["main"] + +permissions: + contents: read + packages: write + +env: + DS_REGISTRY: "ghcr.io/jolicode/docker-starter" + DS_PHP_VERSION: "8.3" + +jobs: + push-images: + name: Push image to registry + runs-on: ubuntu-latest + env: + DS_PHP_VERSION: "8.3" + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - uses: actions/checkout@v4 + + - name: setup-castor + uses: castor-php/setup-castor@v0.1.0 + + - name: Log in to registry + shell: bash + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + + - name: "Build and start the infrastructure" + run: "castor docker:push" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb3275f..0aed02f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,11 +10,13 @@ name: Continuous Integration permissions: contents: read + packages: read env: # Fix for symfony/color detection. We know GitHub Actions can handle it ANSICON: 1 CASTOR_CONTEXT: ci + DS_REGISTRY: "ghcr.io/jolicode/docker-starter" jobs: check-dockerfiles: @@ -41,6 +43,20 @@ jobs: steps: - uses: actions/checkout@v4 + # Install official version of docker that correctly supports from-cache option in docker compose + - name: Set up Docker + uses: crazy-max/ghaction-setup-docker@v3 + with: + set-host: true + + # Docker socket path is different when using setup docker + - name: Set Docker Socket Host + run: echo "DOCKER_SOCKET_PATH=${DOCKER_HOST:5}" >> $GITHUB_ENV + + - name: Log in to registry + shell: bash + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + - name: setup-castor uses: castor-php/setup-castor@v0.1.0 diff --git a/.home/.gitignore b/.home/.gitignore new file mode 100644 index 0000000..78d9101 --- /dev/null +++ b/.home/.gitignore @@ -0,0 +1,2 @@ +/* +!.gitignore diff --git a/README.md b/README.md index c389bf9..c1db385 100644 --- a/README.md +++ b/README.md @@ -957,6 +957,128 @@ services: +### How to use a docker registry to cache images layer + +
+ +Read the cookbook + +You can use a docker registry to cache images layer, it can be useful to speed up the build process during the CI and +local development. + +First you need a docker registry, in following examples we will use the GitHub registry (ghcr.io). + +Then add the registry to the context variable of the `castor.php` file: + +```php +function create_default_variables(): Context +{ + return [ + // [...] + 'registry' => 'ghcr.io/your-organization/your-project', + ]; +} +``` + +Once you have the registry, you can push the images to the registry: + +```bash +castor docker:push +``` + +> [!WARNING] Pushing images cache from a dev environment to a registry is not recommended, as cache is highly sensitive +> to the environment and may not be compatible with other environments. It is recommended to push the cache from the CI +> environment. + +This command will generate a bake file with the images to push from the `cache_from` directive of the `docker-compose.yml` file. +If you want to add more images to push, you can add the `cache_from` directive to them. + +```yaml +services: + my-service: + build: + cache_from: + - "type=registry,ref=${REGISTRY:-}/my-service:cache" +``` +
+ +### How to use cached images in a GitHub action + +
+ +Read the cookbook + +If you are using a GitHub action to build your images, you can use the cached images from the registry to speed up the build process. +However there are few steps to make it works nicely due to the docker binary limitations in GitHub actions. + +#### Pushing images to the registry from a GitHub action + +To push images to the registry in a github action you will need to do this : + +1. Ensure that the github token have the `write:packages` scope. + +```yaml +permissions: + contents: read + packages: write +``` + + +2. Install Docker buildx in the github action + +```yaml + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 +``` + +3. Login to the registry + +```yaml + - name: Log in to registry + shell: bash + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin +``` + +#### Using the cached images in GitHub action + +Images layers built when using the Docker Buildx will have a different hash than the one built with the classic Docker build. +Then you will need to use a more recent version of the Docker binary to use the cached images by either: + +* Use buildx in each GitHub action workflow step + +```yaml + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 +``` + +* Use the official Docker binary +```yaml + # Install official version of docker that correctly supports from-cache option in docker compose + - name: Set up Docker + uses: crazy-max/ghaction-setup-docker@v3 + with: + set-host: true + + # Docker socket path is different when using setup docker + - name: Set Docker Socket Host + run: echo "DOCKER_SOCKET_PATH=${DOCKER_HOST:5}" >> $GITHUB_ENV +``` + +The second option is faster (there is no need to transfer images between buildx and local docker), but it is not +officially supported by GitHub actions and may break in the future. + +* Login to the registry + +```yaml + - name: Log in to registry + shell: bash + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin +``` + +By default images are private in the GitHub registry, you will need to login to the registry to pull the images. + +
+ ## Credits - Created at [JoliCode](https://jolicode.com/) diff --git a/castor.php b/castor.php index 429fbb2..c2d0333 100644 --- a/castor.php +++ b/castor.php @@ -35,6 +35,7 @@ function create_default_variables(): array "www.{$projectName}.{$tld}", ], 'php_version' => $_SERVER['DS_PHP_VERSION'] ?? '8.3', + 'registry' => $_SERVER['DS_REGISTRY'] ?? null, ]; } diff --git a/infrastructure/docker/docker-compose.builder.yml b/infrastructure/docker/docker-compose.builder.yml index b4f70d9..d4d3bfc 100644 --- a/infrastructure/docker/docker-compose.builder.yml +++ b/infrastructure/docker/docker-compose.builder.yml @@ -1,13 +1,13 @@ -volumes: - builder-data: {} - services: builder: build: context: services/php target: builder + cache_from: + - "type=registry,ref=${REGISTRY:-}/builder:cache" depends_on: - postgres + user: "${USER_ID}:${USER_ID}" environment: - COMPOSER_MEMORY_LIMIT=-1 - UID=${USER_ID} @@ -17,9 +17,9 @@ services: - CONTINUOUS_INTEGRATION # Travis CI, Cirrus CI - BUILD_NUMBER # Jenkins, TeamCity - RUN_ID # TaskCluster, dsari + - HOME=/home/app volumes: - - "builder-data:/home/app" - - "${COMPOSER_CACHE_DIR}:/home/app/.composer/cache" - "../..:/var/www:cached" + - "../../.home:/home/app:cached" profiles: - default diff --git a/infrastructure/docker/docker-compose.worker.yml b/infrastructure/docker/docker-compose.worker.yml index 135f4c6..df7cea5 100644 --- a/infrastructure/docker/docker-compose.worker.yml +++ b/infrastructure/docker/docker-compose.worker.yml @@ -4,11 +4,17 @@ x-services-templates: build: context: services/php target: worker + cache_from: + - "${REGISTRY:-}/worker:cache" depends_on: - postgres #- rabbitmq + user: "${USER_ID}:${USER_ID}" volumes: - "../..:/var/www:cached" + - "../../.home:/home/app:cached" + environment: + - HOME=/home/app profiles: - default - worker diff --git a/infrastructure/docker/docker-compose.yml b/infrastructure/docker/docker-compose.yml index 67b3d09..e6ffd43 100644 --- a/infrastructure/docker/docker-compose.yml +++ b/infrastructure/docker/docker-compose.yml @@ -5,7 +5,7 @@ services: router: build: services/router volumes: - - "/var/run/docker.sock:/var/run/docker.sock" + - "${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock" - "./services/router/certs:/etc/ssl/certs" network_mode: host profiles: @@ -15,12 +15,17 @@ services: build: context: services/php target: frontend + cache_from: + - "type=registry,ref=${REGISTRY:-}/frontend:cache" + user: "${USER_ID}:${USER_ID}" depends_on: - postgres volumes: - "../..:/var/www:cached" + - "../../.home:/home/app:cached" environment: - "PHP_VERSION=${PHP_VERSION}" + - HOME=/home/app profiles: - default labels: diff --git a/infrastructure/docker/services/php/Dockerfile b/infrastructure/docker/services/php/Dockerfile index 546c1a6..d126dcf 100644 --- a/infrastructure/docker/services/php/Dockerfile +++ b/infrastructure/docker/services/php/Dockerfile @@ -40,20 +40,19 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* -# Fake user to maps with the one on the host -ARG USER_ID -COPY entrypoint / -RUN addgroup --gid $USER_ID app && \ - adduser --system --uid $USER_ID --home /home/app --shell /bin/bash app && \ - curl -Ls https://github.com/tianon/gosu/releases/download/1.17/gosu-amd64 | \ - install /dev/stdin /usr/local/bin/gosu && \ - sed "s/{{ application_user }}/app/g" -i /entrypoint - # Configuration COPY base/php-configuration /etc/php/${PHP_VERSION} +# Install a fake sudo command +# This is commented out by default because it exposes a security risk if you use this image in production, but it may be useful for development +# Use it at your own risk +#COPY base/sudo.sh /usr/local/bin/sudo +#RUN curl -L https://github.com/tianon/gosu/releases/download/1.16/gosu-amd64 -o /usr/local/bin/gosu && \ +# chmod u+s /usr/local/bin/gosu && \ +# chmod +x /usr/local/bin/gosu && \ +# chmod +x /usr/local/bin/sudo + WORKDIR /var/www -ENTRYPOINT [ "/entrypoint" ] FROM php-base as frontend @@ -70,14 +69,13 @@ RUN useradd -s /bin/false nginx COPY frontend/php-configuration /etc/php/${PHP_VERSION} COPY frontend/etc/nginx/. /etc/nginx/ -COPY frontend/etc/service/. /etc/service/ RUN phpenmod app-default \ && phpenmod app-fpm EXPOSE 80 -CMD ["runsvdir", "-P", "/etc/service"] +CMD ["runsvdir", "-P", "/var/www/infrastructure/docker/services/php/frontend/etc/service"] FROM php-base as worker @@ -104,16 +102,11 @@ RUN apt-get update \ # Config COPY builder/etc/. /etc/ COPY builder/php-configuration /etc/php/${PHP_VERSION} -RUN adduser app sudo \ - && mkdir /var/log/php \ - && chmod 777 /var/log/php \ - && phpenmod app-default \ - && phpenmod app-builder + +RUN phpenmod app-default && phpenmod app-builder # Composer COPY --from=composer/composer:2.7.1 /usr/bin/composer /usr/bin/composer -RUN mkdir -p "/home/app/.composer/cache" \ - && chown app: /home/app/.composer -R # Third party tools ENV PATH="$PATH:/var/www/tools/bin" diff --git a/infrastructure/docker/services/php/base/sudo.sh b/infrastructure/docker/services/php/base/sudo.sh new file mode 100644 index 0000000..cdbf17a --- /dev/null +++ b/infrastructure/docker/services/php/base/sudo.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +export GOSU_PLEASE_LET_ME_BE_COMPLETELY_INSECURE_I_GET_TO_KEEP_ALL_THE_PIECES="I've seen things you people wouldn't believe. Attack ships on fire off the shoulder of Orion. I watched C-beams glitter in the dark near the Tannhäuser Gate. All those moments will be lost in time, like tears in rain. Time to die." +exec gosu 0:0 $@ diff --git a/infrastructure/docker/services/php/entrypoint b/infrastructure/docker/services/php/entrypoint deleted file mode 100755 index 1f15e84..0000000 --- a/infrastructure/docker/services/php/entrypoint +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -set -e -set -u - -if [ $(id -u) != 0 ]; then - echo "Running this image as non root is not allowed" - exit 1 -fi - -: "${UID:=0}" -: "${GID:=${UID}}" - -if [ "$#" = 0 ]; then - set -- "$(command -v bash 2>/dev/null || command -v sh)" -l -fi - -if [ "$UID" != 0 ]; then - usermod -u "$UID" "{{ application_user }}" >/dev/null 2>/dev/null && { - groupmod -g "$GID" "{{ application_user }}" >/dev/null 2>/dev/null || - usermod -a -G "$GID" "{{ application_user }}" >/dev/null 2>/dev/null - } - set -- gosu "${UID}:${GID}" "${@}" -fi - -exec "$@" diff --git a/infrastructure/docker/services/php/frontend/etc/nginx/nginx.conf b/infrastructure/docker/services/php/frontend/etc/nginx/nginx.conf index 2d69c60..d3d7efa 100644 --- a/infrastructure/docker/services/php/frontend/etc/nginx/nginx.conf +++ b/infrastructure/docker/services/php/frontend/etc/nginx/nginx.conf @@ -1,5 +1,4 @@ -user nginx; -pid /var/run/nginx.pid; +pid /tmp/nginx.pid; daemon off; error_log /proc/self/fd/2; include /etc/nginx/modules-enabled/*.conf; @@ -25,6 +24,12 @@ http { gzip_http_version 1.1; gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml; + client_body_temp_path /tmp/nginx-client_body_temp_path; + fastcgi_temp_path /tmp/nginx-fastcgi_temp_path; + proxy_temp_path /tmp/nginx-proxy_temp_path; + scgi_temp_path /tmp/nginx-scgi_temp_path; + uwsgi_temp_path /tmp/nginx-uwsgi_temp_path; + server { listen 0.0.0.0:80; root /var/www/application/public; diff --git a/infrastructure/docker/services/php/frontend/etc/service/.gitignore b/infrastructure/docker/services/php/frontend/etc/service/.gitignore new file mode 100644 index 0000000..8e8a4f9 --- /dev/null +++ b/infrastructure/docker/services/php/frontend/etc/service/.gitignore @@ -0,0 +1 @@ +*/supervise diff --git a/infrastructure/docker/services/php/frontend/php-configuration/fpm/php-fpm.conf b/infrastructure/docker/services/php/frontend/php-configuration/fpm/php-fpm.conf index 38b5901..3b74d34 100644 --- a/infrastructure/docker/services/php/frontend/php-configuration/fpm/php-fpm.conf +++ b/infrastructure/docker/services/php/frontend/php-configuration/fpm/php-fpm.conf @@ -1,11 +1,8 @@ [global] -pid = /var/run/php-fpm.pid error_log = /proc/self/fd/2 daemonize = no [www] -user = app -group = app listen = 127.0.0.1:9000 pm = dynamic pm.max_children = 25