Merge branch 'codex/TUR-174-backend-image-scaling' into 'master'

TUR-174: Backend image scaling

Closes TUR-174

See merge request turniere/turniere-backend!75

.gitlab-ci.yml

@@ -57,13 +57,15 @@ rails e2e spec:
       alias: app
       variables:
         TURNIERE_DISABLE_EMAIL_DELIVERY: "1"
+        WEB_CONCURRENCY: "1"
+        RAILS_MAX_THREADS: "5"
+        RAILS_DB_POOL: "5"
+        SIDEKIQ_CONCURRENCY: "5"
       command:
         - bundle
         - exec
         - rails
         - s
-        - -p
-        - "3000"
         - -b
         - 0.0.0.0
   variables:
@@ -71,6 +73,7 @@ rails e2e spec:
     POSTGRES_DB: turniere_blackbox
     POSTGRES_USER: turniere
     POSTGRES_PASSWORD: turniere
+    DATABASE_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
     POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
     POSTGRES_USERNAME: turniere
     SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod
@@ -112,13 +115,15 @@ rails load smoke:
       alias: app
       variables:
         TURNIERE_DISABLE_EMAIL_DELIVERY: "1"
+        WEB_CONCURRENCY: "1"
+        RAILS_MAX_THREADS: "5"
+        RAILS_DB_POOL: "5"
+        SIDEKIQ_CONCURRENCY: "5"
       command:
         - bundle
         - exec
         - rails
         - s
-        - -p
-        - "3000"
         - -b
         - 0.0.0.0
   variables:
@@ -126,6 +131,7 @@ rails load smoke:
     POSTGRES_DB: turniere_blackbox
     POSTGRES_USER: turniere
     POSTGRES_PASSWORD: turniere
+    DATABASE_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
     POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
     POSTGRES_USERNAME: turniere
     SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod
@@ -181,6 +187,7 @@ rails follow sync e2e spec:
     - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA
       alias: source-app
       variables:
+        DATABASE_URL: postgres://turniere:turniere@source-postgres:5432/turniere_source_blackbox
         POSTGRES_URL: postgres://turniere:turniere@source-postgres:5432/turniere_source_blackbox
         POSTGRES_USERNAME: turniere
         POSTGRES_PASSWORD: turniere
@@ -193,18 +200,21 @@ rails follow sync e2e spec:
         no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1
         RAILS_LOG_TO_STDOUT: "1"
         RAILS_SERVE_STATIC_FILES: "1"
+        WEB_CONCURRENCY: "1"
+        RAILS_MAX_THREADS: "5"
+        RAILS_DB_POOL: "5"
+        SIDEKIQ_CONCURRENCY: "5"
       command:
         - bundle
         - exec
         - rails
         - s
-        - -p
-        - "3000"
         - -b
         - 0.0.0.0
     - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA
       alias: follower-app
       variables:
+        DATABASE_URL: postgres://turniere:turniere@follower-postgres:5432/turniere_follower_blackbox
         POSTGRES_URL: postgres://turniere:turniere@follower-postgres:5432/turniere_follower_blackbox
         POSTGRES_USERNAME: turniere
         POSTGRES_PASSWORD: turniere
@@ -217,13 +227,15 @@ rails follow sync e2e spec:
         no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1
         RAILS_LOG_TO_STDOUT: "1"
         RAILS_SERVE_STATIC_FILES: "1"
+        WEB_CONCURRENCY: "1"
+        RAILS_MAX_THREADS: "5"
+        RAILS_DB_POOL: "5"
+        SIDEKIQ_CONCURRENCY: "5"
       command:
         - bundle
         - exec
         - rails
         - s
-        - -p
-        - "3000"
         - -b
         - 0.0.0.0
   variables:

Dockerfile

@@ -81,7 +81,7 @@ COPY config/environments/production.rb /app/config/environments/beta.rb
 
 EXPOSE 3000
 
-CMD ["sh", "-lc", "bundle exec rails db:migrate && bundle exec rails s -p 3000 -b 0.0.0.0"]
+CMD ["sh", "-lc", "bundle exec rails db:migrate && bundle exec rails s -b 0.0.0.0"]
 
 FROM runtime-base AS test
 

config/database.yml

@@ -6,7 +6,7 @@
 #
 default: &default
   adapter: sqlite3
-  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+  pool: <%= ENV.fetch("RAILS_DB_POOL", ENV.fetch("RAILS_MAX_THREADS", 5)) %>
   timeout: 5000
 
 development:
@@ -21,10 +21,12 @@ test:
   database: db/test.sqlite3
 
 production: &production
-  url: <%= ENV.fetch("POSTGRES_URL", nil) %>
+  adapter: postgresql
+  encoding: unicode
+  url: <%= ENV["DATABASE_URL"] || ENV["POSTGRES_URL"] %>
   username: <%= ENV.fetch("POSTGRES_USERNAME", nil) %>
   password: <%= ENV.fetch("POSTGRES_PASSWORD", nil) %>
-  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+  pool: <%= ENV.fetch("RAILS_DB_POOL", ENV.fetch("RAILS_MAX_THREADS", 5)) %>
 
 beta:
   # just use the production settings

config/puma.rb

@@ -6,8 +6,9 @@
 # the maximum value specified for Puma. Default is set to 5 threads for minimum
 # and maximum; this matches the default thread size of Active Record.
 #
-threads_count = ENV.fetch('RAILS_MAX_THREADS', 5)
+max_threads_count = ENV.fetch('RAILS_MAX_THREADS', 5)
-threads threads_count, threads_count
+min_threads_count = ENV.fetch('RAILS_MIN_THREADS', max_threads_count)
+threads min_threads_count, max_threads_count
 
 # Specifies the `port` that Puma will listen on to receive requests; default is 3000.
 #
@@ -15,7 +16,7 @@ port        ENV.fetch('PORT', 3000)
 
 # Specifies the `environment` that Puma will run in.
 #
-environment ENV.fetch('RAILS_ENV', 'development')
+environment ENV.fetch('RAILS_ENV', 'production')
 
 # Specifies the number of `workers` to boot in clustered mode.
 # Workers are forked webserver processes. If using threads and workers together
@@ -23,14 +24,14 @@ environment ENV.fetch('RAILS_ENV', 'development')
 # Workers do not work on JRuby or Windows (both of which do not support
 # processes).
 #
-# workers ENV.fetch("WEB_CONCURRENCY") { 2 }
+workers ENV.fetch('WEB_CONCURRENCY', 2)
 
 # Use the `preload_app!` method when specifying a `workers` number.
 # This directive tells Puma to first boot the application and load code
 # before forking the application. This takes advantage of Copy On Write
 # process behavior so workers use less memory.
 #
-# preload_app!
+preload_app!
 
 # Allow puma to be restarted by `rails restart` command.
 plugin :tmp_restart

config/sidekiq.yml

@@ -0,0 +1,2 @@
+---
+:concurrency: <%= ENV.fetch("SIDEKIQ_CONCURRENCY", 5) %>

docker-compose.blackbox-follow.yml

@@ -37,6 +37,12 @@ services:
       source-postgres:
         condition: service_healthy
     environment:
+      RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production}
+      WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-1}
+      RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5}
+      RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5}
+      SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5}
+      DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@source-postgres:5432/${TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB:-turniere_source_blackbox}
       POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@source-postgres:5432/${TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB:-turniere_source_blackbox}
       POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}
       POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}
@@ -59,6 +65,12 @@ services:
       follower-postgres:
         condition: service_healthy
     environment:
+      RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production}
+      WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-1}
+      RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5}
+      RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5}
+      SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5}
+      DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@follower-postgres:5432/${TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB:-turniere_follower_blackbox}
       POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@follower-postgres:5432/${TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB:-turniere_follower_blackbox}
       POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}
       POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}

docker-compose.blackbox.yml

@@ -19,6 +19,12 @@ services:
       postgres:
         condition: service_healthy
     environment:
+      RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production}
+      WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-1}
+      RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5}
+      RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5}
+      SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5}
+      DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@postgres:5432/${TURNIERE_BLACKBOX_POSTGRES_DB:-turniere_blackbox}
       POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@postgres:5432/${TURNIERE_BLACKBOX_POSTGRES_DB:-turniere_blackbox}
       POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}
       POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}

spec/config/scaling_env_spec.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'yaml'
+
+RSpec.describe 'runtime scaling configuration' do
+  around do |example|
+    with_env(
+      'DATABASE_URL' => nil,
+      'POSTGRES_URL' => nil,
+      'PORT' => nil,
+      'RAILS_DB_POOL' => nil,
+      'RAILS_ENV' => nil,
+      'RAILS_MAX_THREADS' => nil,
+      'RAILS_MIN_THREADS' => nil,
+      'SIDEKIQ_CONCURRENCY' => nil,
+      'WEB_CONCURRENCY' => nil
+    ) { example.run }
+  end
+
+  it 'configures Puma workers and threads from environment variables' do
+    with_env(
+      'PORT' => '4001',
+      'RAILS_ENV' => 'production',
+      'RAILS_MAX_THREADS' => '7',
+      'RAILS_MIN_THREADS' => '2',
+      'WEB_CONCURRENCY' => '4'
+    ) do
+      config = evaluate_puma_config
+
+      expect(config.fetch(:threads)).to eq(%w[2 7])
+      expect(config.fetch(:workers)).to eq('4')
+      expect(config.fetch(:preload_app)).to be(true)
+      expect(config.fetch(:port)).to eq('4001')
+      expect(config.fetch(:environment)).to eq('production')
+      expect(config.fetch(:plugin)).to eq(:tmp_restart)
+    end
+  end
+
+  it 'uses safe Puma defaults' do
+    config = evaluate_puma_config
+
+    expect(config.fetch(:threads)).to eq([5, 5])
+    expect(config.fetch(:workers)).to eq(2)
+    expect(config.fetch(:port)).to eq(3000)
+    expect(config.fetch(:environment)).to eq('production')
+  end
+
+  it 'configures the production database pool from RAILS_DB_POOL' do
+    with_env(
+      'DATABASE_URL' => 'postgres://turniere:secret@postgres/turniere',
+      'RAILS_DB_POOL' => '9',
+      'RAILS_MAX_THREADS' => '7'
+    ) do
+      config = database_config.fetch('production')
+
+      expect(config.fetch('adapter')).to eq('postgresql')
+      expect(config.fetch('encoding')).to eq('unicode')
+      expect(config.fetch('url')).to eq('postgres://turniere:secret@postgres/turniere')
+      expect(config.fetch('pool')).to eq(9)
+    end
+  end
+
+  it 'falls back to RAILS_MAX_THREADS for the database pool and POSTGRES_URL for legacy deployments' do
+    with_env(
+      'POSTGRES_URL' => 'postgres://turniere:secret@legacy-postgres/turniere',
+      'RAILS_MAX_THREADS' => '6'
+    ) do
+      config = database_config.fetch('production')
+
+      expect(config.fetch('url')).to eq('postgres://turniere:secret@legacy-postgres/turniere')
+      expect(config.fetch('pool')).to eq(6)
+    end
+  end
+
+  it 'configures Sidekiq concurrency from SIDEKIQ_CONCURRENCY' do
+    with_env('SIDEKIQ_CONCURRENCY' => '10') do
+      expect(sidekiq_config).to include(':concurrency: 10')
+    end
+
+    expect(sidekiq_config).to include(':concurrency: 5')
+  end
+
+  def evaluate_puma_config
+    probe = puma_config_probe_class.new
+    probe.instance_eval(File.read(File.join(root, 'config/puma.rb')), File.join(root, 'config/puma.rb'))
+    probe.config
+  end
+
+  def database_config
+    YAML.safe_load(ERB.new(File.read(File.join(root, 'config/database.yml'))).result, aliases: true)
+  end
+
+  def sidekiq_config
+    ERB.new(File.read(File.join(root, 'config/sidekiq.yml'))).result
+  end
+
+  def with_env(values)
+    previous = values.to_h { |key, _value| [key, ENV.fetch(key, nil)] }
+    values.each do |key, value|
+      value.nil? ? ENV.delete(key) : ENV[key] = value
+    end
+
+    yield
+  ensure
+    previous.each do |key, value|
+      value.nil? ? ENV.delete(key) : ENV[key] = value
+    end
+  end
+
+  def root
+    File.expand_path('../..', __dir__)
+  end
+
+  def puma_config_probe_class
+    Class.new do
+      attr_reader :config
+
+      def initialize
+        @config = {}
+      end
+
+      def threads(min_threads, max_threads)
+        @config[:threads] = [min_threads, max_threads]
+      end
+
+      def workers(count)
+        @config[:workers] = count
+      end
+
+      def preload_app!
+        @config[:preload_app] = true
+      end
+
+      def port(value)
+        @config[:port] = value
+      end
+
+      def environment(value)
+        @config[:environment] = value
+      end
+
+      def plugin(name)
+        @config[:plugin] = name
+      end
+    end
+  end
+end

tasks.py

@@ -470,12 +470,19 @@ def _spec_shards(node_total):
 
 
 def _shared_production_env():
-    return _env(
+    database_url = os.environ.get(
-        RAILS_ENV="production",
+        "DATABASE_URL",
-        POSTGRES_URL=os.environ.get(
+        os.environ.get(
             "POSTGRES_URL",
             f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@postgres:5432/{BLACKBOX_DB_NAME}",
         ),
+    )
+    rails_max_threads = os.environ.get("RAILS_MAX_THREADS", "5")
+
+    return _env(
+        RAILS_ENV="production",
+        DATABASE_URL=database_url,
+        POSTGRES_URL=database_url,
         POSTGRES_USERNAME=os.environ.get("POSTGRES_USERNAME", BLACKBOX_DB_USER),
         POSTGRES_PASSWORD=os.environ.get("POSTGRES_PASSWORD", BLACKBOX_DB_PASSWORD),
         SECRET_KEY_BASE=os.environ.get("SECRET_KEY_BASE", BLACKBOX_SECRET_KEY_BASE),
@@ -484,13 +491,20 @@ def _shared_production_env():
         TURNIERE_DISABLE_EMAIL_DELIVERY=os.environ.get("TURNIERE_DISABLE_EMAIL_DELIVERY", "1"),
         RAILS_LOG_TO_STDOUT=os.environ.get("RAILS_LOG_TO_STDOUT", "1"),
         RAILS_SERVE_STATIC_FILES=os.environ.get("RAILS_SERVE_STATIC_FILES", "1"),
+        WEB_CONCURRENCY=os.environ.get("WEB_CONCURRENCY", "2"),
+        RAILS_MAX_THREADS=rails_max_threads,
+        RAILS_DB_POOL=os.environ.get("RAILS_DB_POOL", rails_max_threads),
+        SIDEKIQ_CONCURRENCY=os.environ.get("SIDEKIQ_CONCURRENCY", "5"),
     )
 
 
 def _production_env_for(db_host, db_name):
+    database_url = f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@{db_host}:5432/{db_name}"
+
     return _env(
         RAILS_ENV="production",
-        POSTGRES_URL=f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@{db_host}:5432/{db_name}",
+        DATABASE_URL=database_url,
+        POSTGRES_URL=database_url,
         POSTGRES_USERNAME=BLACKBOX_DB_USER,
         POSTGRES_PASSWORD=BLACKBOX_DB_PASSWORD,
         SECRET_KEY_BASE=BLACKBOX_SECRET_KEY_BASE,
@@ -499,6 +513,10 @@ def _production_env_for(db_host, db_name):
         TURNIERE_DISABLE_EMAIL_DELIVERY="1",
         RAILS_LOG_TO_STDOUT="1",
         RAILS_SERVE_STATIC_FILES="1",
+        WEB_CONCURRENCY=os.environ.get("WEB_CONCURRENCY", "2"),
+        RAILS_MAX_THREADS=os.environ.get("RAILS_MAX_THREADS", "5"),
+        RAILS_DB_POOL=os.environ.get("RAILS_DB_POOL", os.environ.get("RAILS_MAX_THREADS", "5")),
+        SIDEKIQ_CONCURRENCY=os.environ.get("SIDEKIQ_CONCURRENCY", "5"),
     )
 
 
@@ -1094,6 +1112,101 @@ def docker_build_all(c):
     docker_build_test(c)
 
 
+def _assert_output_match(label, output, pattern):
+    if re.search(pattern, output) is not None:
+        return
+
+    tail = output[-4000:] if output else "<empty output>"
+    raise RuntimeError(f"{label} did not match {pattern!r}.\nLast output:\n{tail}")
+
+
+def _blackbox_app_exec(env, *command):
+    result = _compose_run(["exec", "-T", "app", *command], env=env, capture_output=True)
+    output = "\n".join(part for part in (result.stdout, result.stderr) if part)
+    return output.strip()
+
+
+def _assert_blackbox_scaling_config(env, expected):
+    logs = _compose_run(["logs", "app"], env=env, capture_output=True).stdout
+
+    _assert_output_match("Puma worker count", logs, rf"Workers:\s+{re.escape(expected['web_concurrency'])}\b")
+    _assert_output_match("Puma max threads", logs, rf"Max threads:\s+{re.escape(expected['rails_max_threads'])}\b")
+    _assert_output_match("Puma environment", logs, r"Environment:\s+production\b")
+
+    pool_size = _blackbox_app_exec(env, "bundle", "exec", "rails", "runner", "puts ActiveRecord::Base.connection_pool.size")
+    if pool_size.splitlines()[-1] != expected["rails_db_pool"]:
+        raise RuntimeError(f"Expected DB pool {expected['rails_db_pool']}, got {pool_size!r}")
+
+    sidekiq_config = _blackbox_app_exec(
+        env,
+        "ruby",
+        "-rerb",
+        "-e",
+        "puts ERB.new(File.read('config/sidekiq.yml')).result",
+    )
+    expected_sidekiq = f":concurrency: {expected['sidekiq_concurrency']}"
+    if expected_sidekiq not in sidekiq_config:
+        raise RuntimeError(f"Expected Sidekiq config {expected_sidekiq!r}, got {sidekiq_config!r}")
+
+
+@task(name="docker-blackbox-scaling-config")
+def docker_blackbox_scaling_config(
+    c,
+    host_port="3004",
+    app_image=PRODUCTION_TAG,
+    runner_image=TEST_TAG,
+    postgres_image=BLACKBOX_POSTGRES_IMAGE,
+    build=True,
+):
+    """Build once, boot the production image with two scaling env sets, and assert runtime config."""
+    if build:
+        docker_build_production(c, tag=app_image)
+
+    cases = [
+        {
+            "label": "medium",
+            "web_concurrency": "2",
+            "rails_max_threads": "3",
+            "rails_db_pool": "4",
+            "sidekiq_concurrency": "6",
+        },
+        {
+            "label": "event",
+            "web_concurrency": "3",
+            "rails_max_threads": "4",
+            "rails_db_pool": "5",
+            "sidekiq_concurrency": "7",
+        },
+    ]
+
+    for expected in cases:
+        env = _compose_env(
+            host_port=host_port,
+            app_image=app_image,
+            runner_image=runner_image,
+            postgres_image=postgres_image,
+            expect_profiling="false",
+        )
+        env.update(
+            TURNIERE_BLACKBOX_RAILS_ENV="production",
+            TURNIERE_BLACKBOX_WEB_CONCURRENCY=expected["web_concurrency"],
+            TURNIERE_BLACKBOX_RAILS_MAX_THREADS=expected["rails_max_threads"],
+            TURNIERE_BLACKBOX_RAILS_DB_POOL=expected["rails_db_pool"],
+            TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY=expected["sidekiq_concurrency"],
+        )
+
+        _print_header(f"Scaling config case: {expected['label']}")
+        try:
+            _compose_run(["down", "-v", "--remove-orphans"], env=env, check=False)
+            _compose_run(["up", "-d", "postgres", "app"], env=env)
+            _wait_for_http(base_url=f"http://127.0.0.1:{host_port}", timeout=120)
+            _assert_blackbox_scaling_config(env, expected)
+        finally:
+            _compose_run(["down", "-v", "--remove-orphans"], env=env, check=False)
+
+    print("Scaling config changed across boots without rebuilding the production image.")
+
+
 @task(name="docker-blackbox-up")
 def docker_blackbox_up(
     c,