Merge branch 'codex/buildx-multistage-multiarch'

.gitlab-ci.yml

@@ -6,6 +6,8 @@ stages:
 
 variables:
   DOCKER_IMAGE_ENVS: "production test"
+  DOCKERFILE_PATH: "Dockerfile"
+  DOCKER_BUILD_TARGETS: "production=production test=test"
 
 include:
   - project: 'turniere/turniere-infra'
@@ -13,10 +15,10 @@ include:
 
 rails spec:
   stage: test
-  needs:
-    - job: build_image
   tags:
     - docker
+  needs:
+    - job: build_image
   image: $CI_REGISTRY_IMAGE/test/commits:$CI_COMMIT_SHA
   parallel:
     matrix:
@@ -43,10 +45,10 @@ rails spec:
 
 rails e2e spec:
   stage: e2e
-  needs:
-    - job: build_image
   tags:
     - docker
+  needs:
+    - job: build_image
   image: $CI_REGISTRY_IMAGE/test/commits:$CI_COMMIT_SHA
   services:
     - name: postgres:16
@@ -96,10 +98,10 @@ rails e2e spec:
 
 rails follow sync e2e spec:
   stage: e2e
-  needs:
-    - job: build_image
   tags:
     - docker
+  needs:
+    - job: build_image
   image: $CI_REGISTRY_IMAGE/test/commits:$CI_COMMIT_SHA
   services:
     - name: postgres:16

Dockerfile

@@ -0,0 +1,105 @@
+# syntax=docker/dockerfile:1.7
+
+ARG RUBY_VERSION=3.1.2
+ARG BUNDLER_VERSION=2.3.13
+
+FROM ruby:${RUBY_VERSION}-slim AS runtime-base
+
+ARG BUNDLER_VERSION
+
+ENV LANG=C.UTF-8 \
+    BUNDLE_JOBS=4 \
+    BUNDLE_RETRY=3 \
+    BUNDLE_PATH=/app/vendor/bundle \
+    BUNDLE_APP_CONFIG=/app/vendor/bundle
+
+WORKDIR /app
+
+RUN apt-get update -qq \
+    && apt-get install -y --no-install-recommends \
+        libpq5 \
+        nodejs \
+        openssl \
+    && rm -rf /var/lib/apt/lists/* \
+    && gem install bundler -v ${BUNDLER_VERSION} \
+    && gem install tzinfo-data
+
+FROM runtime-base AS build-base
+
+RUN apt-get update -qq \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        git \
+        libpq-dev \
+        libsqlite3-dev \
+        pkg-config \
+        python3 \
+        python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY Gemfile Gemfile.lock ./
+
+FROM build-base AS bundle-production
+
+RUN bundle config set deployment 'true' \
+    && bundle config set without 'development test' \
+    && bundle install \
+    && rm -rf /usr/local/bundle/cache/*.gem
+
+FROM build-base AS bundle-test
+
+RUN bundle config set with 'development test' \
+    && bundle install \
+    && rm -rf /usr/local/bundle/cache/*.gem
+
+FROM runtime-base AS production
+
+ENV RAILS_ENV=production
+
+COPY --from=bundle-production /usr/local/bundle /usr/local/bundle
+COPY --from=bundle-production /app/vendor/bundle /app/vendor/bundle
+COPY Gemfile Gemfile.lock /app/
+COPY app /app/app
+COPY bin /app/bin
+COPY config /app/config
+COPY db /app/db
+COPY public /app/public
+COPY script /app/script
+COPY config.ru Rakefile /app/
+# Duplicate production environment to beta environment
+COPY config/environments/production.rb /app/config/environments/beta.rb
+
+EXPOSE 3000
+
+CMD ["sh", "-lc", "bundle exec rails db:migrate && bundle exec rails s -p 3000 -b 0.0.0.0"]
+
+FROM runtime-base AS test
+
+ENV RAILS_ENV=test
+
+RUN apt-get update -qq \
+    && apt-get install -y --no-install-recommends \
+        libsqlite3-0 \
+        python3 \
+        python3-pip \
+    && rm -rf /var/lib/apt/lists/* \
+    && python3 -m pip install --no-cache-dir invoke
+
+COPY --from=bundle-test /usr/local/bundle /usr/local/bundle
+COPY --from=bundle-test /app/vendor/bundle /app/vendor/bundle
+COPY Gemfile Gemfile.lock /app/
+COPY app /app/app
+COPY bin /app/bin
+COPY config /app/config
+COPY db /app/db
+COPY public /app/public
+COPY script /app/script
+COPY config.ru Rakefile /app/
+COPY config/environments/production.rb /app/config/environments/beta.rb
+COPY tasks.py /app/tasks.py
+COPY e2e /app/e2e
+COPY lib /app/lib
+COPY spec /app/spec
+COPY .rspec /app/.rspec
+
+CMD ["sh", "-lc", "bundle exec rails db:migrate && bundle exec rails spec"]

doc/buildx_migration_notes.md

@@ -0,0 +1,61 @@
+# Buildx Migration Notes
+
+This backend branch temporarily pins the shared GitLab include to the infra branch
+`codex/buildx-multiarch-template` so the new shared image build flow can be tested
+end to end before the infra MR is merged.
+
+After the infra MR is merged:
+
+- remove the temporary `ref:` override from `.gitlab-ci.yml`
+- keep the repo-level variables that describe how this repo maps images to a single
+  multi-stage `Dockerfile`
+
+## Backend pattern
+
+Backend now uses:
+
+- one root `Dockerfile`
+- `DOCKER_IMAGE_ENVS: "production test"`
+- `DOCKERFILE_PATH: "Dockerfile"`
+- `DOCKER_BUILD_TARGETS: "production=production test=test"`
+- shared infra template builds amd64 automatically on push pipelines
+- shared infra template builds arm64 manually on branch pipelines and automatically on default-branch pushes
+
+## turniere-frontend
+
+Frontend can move in two steps.
+
+1. No Dockerfile restructuring required for the shared buildx rollout itself.
+   Its current `docker/production/Dockerfile` already fits the shared template.
+2. To add Raspberry Pi support, verify the current production image really builds on
+   `linux/arm64`.
+   The current file uses `node:16-alpine` and `alpine`; that should be checked in CI.
+
+Recommended frontend CI change after infra is merged:
+
+- keep `DOCKER_IMAGE_ENVS: "production"`
+- no extra platform variable needed if default shared behavior is acceptable
+
+Optional frontend follow-up:
+
+- if you want the same repo shape as backend, replace `docker/production/Dockerfile`
+  with a root multi-stage `Dockerfile`, then add:
+  `DOCKERFILE_PATH: "Dockerfile"` and
+  `DOCKER_BUILD_TARGETS: "production=production"`
+
+## turniere-match
+
+Match also does not need a Dockerfile restructuring for the shared buildx rollout.
+Its current `docker/production/Dockerfile` already matches the default shared
+template contract.
+
+Recommended match CI change after infra is merged:
+
+- keep `DOCKER_IMAGE_ENVS: "production"`
+- no extra platform variable needed if default shared behavior is acceptable
+
+Suggested follow-up:
+
+- run a multi-arch build smoke test for the Alpine-based Python image
+- if packaging or native wheels become slow on arm64, consider a multi-stage
+  Dockerfile with a slimmer runtime image similar to the backend pattern

docker/production/Dockerfile

@@ -1,19 +0,0 @@
-ARG base_commit
-FROM ruby:3.1.2-slim
-ENV RAILS_ENV production
-# RUN apk add --no-cache build-base tzdata sqlite sqlite-dev postgresql-dev git && gem install tzinfo-data
-RUN apt-get update -qq && apt-get install -y openssl git build-essential libpq-dev nodejs && apt-get clean && gem install tzinfo-data
-WORKDIR /app
-COPY Gemfile* /app/
-RUN gem install bundler:2.3.13 && bundle config set deployment 'true' && bundle install
-COPY app /app/app
-COPY bin /app/bin
-COPY config /app/config
-COPY db /app/db
-COPY public /app/public
-COPY script /app/script
-COPY config.ru Rakefile /app/
-# Duplicate production environment to beta environment
-COPY config/environments/production.rb config/environments/beta.rb
-EXPOSE 3000
-CMD bundle exec rails db:migrate && bundle exec rails s -p 3000 -b 0.0.0.0

docker/test/Dockerfile

@@ -1,13 +0,0 @@
-# Build production container locally first with following tag
-ARG base_commit
-FROM registry.gitlab.com/turniere/turniere-backend/production/commits:$base_commit
-WORKDIR /app
-RUN apt-get update -qq && apt-get install -y python3 python3-pip && apt-get clean && python3 -m pip install --no-cache-dir invoke
-RUN bundle config set with 'development test' && bundle install
-COPY tasks.py /app/tasks.py
-COPY e2e /app/e2e
-COPY lib /app/lib
-COPY spec /app/spec
-COPY .rspec /app/.rspec
-ENV RAILS_ENV test
-CMD bundle exec rails db:migrate && bundle exec rails spec

tasks.py

@@ -31,6 +31,9 @@ E2E_ALT_USERNAME = "e2e-alt-user"
 
 PRODUCTION_TAG = "registry.gitlab.com/turniere/turniere-backend/production/commits:local"
 TEST_TAG = "registry.gitlab.com/turniere/turniere-backend/test/commits:local"
+DOCKERFILE_PATH = "Dockerfile"
+PRODUCTION_DOCKER_TARGET = "production"
+TEST_DOCKER_TARGET = "test"
 BLACKBOX_COMPOSE_FILE = "docker-compose.blackbox.yml"
 BLACKBOX_PROJECT = "turniere-blackbox"
 BLACKBOX_FOLLOW_COMPOSE_FILE = "docker-compose.blackbox-follow.yml"
@@ -46,9 +49,10 @@ BLACKBOX_MAILGUN_API_KEY = "blackbox-test-api-key"
 BLACKBOX_MAILGUN_DOMAIN = "blackbox.example.com"
 SPEC_ROOT = Path("spec")
 E2E_SPEC_ROOT = SPEC_ROOT / "e2e" / "http"
-BUNDLER_VERSION = Path("Gemfile.lock").read_text(encoding="utf-8").split("BUNDLED WITH", 1)[1].strip().splitlines()[-1].strip()
 ROOT = Path(__file__).resolve().parent
 DB_DIR = ROOT / "db"
+GEMFILE_LOCK_PATH = ROOT / "Gemfile.lock"
+BUNDLER_VERSION = GEMFILE_LOCK_PATH.read_text(encoding="utf-8").split("BUNDLED WITH", 1)[1].strip().splitlines()[-1].strip()
 
 
 def _env(**overrides):
@@ -228,6 +232,31 @@ def _format_command(command):
     return " ".join(shlex.quote(part) for part in command)
 
 
+def _docker_build_command(tag, target, platforms=None, push=False):
+    if platforms and "," in str(platforms) and not push:
+        raise ValueError("multi-platform buildx builds require push=True")
+
+    if platforms or push:
+        command = [
+            "docker",
+            "buildx",
+            "build",
+            "--target",
+            target,
+            "-t",
+            tag,
+            "-f",
+            DOCKERFILE_PATH,
+        ]
+        if platforms:
+            command.extend(["--platform", platforms])
+        command.append("--push" if push else "--load")
+        command.append(".")
+        return _format_command(command)
+
+    return f"docker build --target {target} -t {tag} -f {DOCKERFILE_PATH} ."
+
+
 def _rspec_command(*paths):
     command = ["bundle", "exec", "rspec", *paths]
     if os.environ.get("CI"):
@@ -815,15 +844,15 @@ def scenario_render_profile(
 
 
 @task(name="docker-build-production")
-def docker_build_production(c, tag=PRODUCTION_TAG):
+def docker_build_production(c, tag=PRODUCTION_TAG, platforms=None, push=False):
     """Build the production Docker image."""
-    c.run(f"docker build -t {tag} -f docker/production/Dockerfile .", pty=True)
+    c.run(_docker_build_command(tag, PRODUCTION_DOCKER_TARGET, platforms=platforms, push=push), pty=True)
 
 
 @task(name="docker-build-test")
-def docker_build_test(c, tag=TEST_TAG):
+def docker_build_test(c, tag=TEST_TAG, platforms=None, push=False):
     """Build the test Docker image."""
-    c.run(f"docker build --build-arg base_commit=local -t {tag} -f docker/test/Dockerfile .", pty=True)
+    c.run(_docker_build_command(tag, TEST_DOCKER_TARGET, platforms=platforms, push=push), pty=True)
 
 
 @task(name="docker-build-all")