diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99034f1..7c37abf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,13 +57,15 @@ rails e2e spec: alias: app variables: TURNIERE_DISABLE_EMAIL_DELIVERY: "1" + WEB_CONCURRENCY: "2" + RAILS_MAX_THREADS: "5" + RAILS_DB_POOL: "5" + SIDEKIQ_CONCURRENCY: "5" command: - bundle - exec - rails - s - - -p - - "3000" - -b - 0.0.0.0 variables: @@ -71,6 +73,7 @@ rails e2e spec: POSTGRES_DB: turniere_blackbox POSTGRES_USER: turniere POSTGRES_PASSWORD: turniere + DATABASE_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox POSTGRES_USERNAME: turniere SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod @@ -112,13 +115,15 @@ rails load smoke: alias: app variables: TURNIERE_DISABLE_EMAIL_DELIVERY: "1" + WEB_CONCURRENCY: "2" + RAILS_MAX_THREADS: "5" + RAILS_DB_POOL: "5" + SIDEKIQ_CONCURRENCY: "5" command: - bundle - exec - rails - s - - -p - - "3000" - -b - 0.0.0.0 variables: @@ -126,6 +131,7 @@ rails load smoke: POSTGRES_DB: turniere_blackbox POSTGRES_USER: turniere POSTGRES_PASSWORD: turniere + DATABASE_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox POSTGRES_USERNAME: turniere SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod @@ -181,6 +187,7 @@ rails follow sync e2e spec: - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA alias: source-app variables: + DATABASE_URL: postgres://turniere:turniere@source-postgres:5432/turniere_source_blackbox POSTGRES_URL: postgres://turniere:turniere@source-postgres:5432/turniere_source_blackbox POSTGRES_USERNAME: turniere POSTGRES_PASSWORD: turniere @@ -193,18 +200,21 @@ rails follow sync e2e spec: no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 RAILS_LOG_TO_STDOUT: "1" RAILS_SERVE_STATIC_FILES: "1" + WEB_CONCURRENCY: "2" + RAILS_MAX_THREADS: "5" + RAILS_DB_POOL: "5" + SIDEKIQ_CONCURRENCY: "5" command: - bundle - exec - rails - s - - -p - - "3000" - -b - 0.0.0.0 - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA alias: follower-app variables: + DATABASE_URL: postgres://turniere:turniere@follower-postgres:5432/turniere_follower_blackbox POSTGRES_URL: postgres://turniere:turniere@follower-postgres:5432/turniere_follower_blackbox POSTGRES_USERNAME: turniere POSTGRES_PASSWORD: turniere @@ -217,13 +227,15 @@ rails follow sync e2e spec: no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 RAILS_LOG_TO_STDOUT: "1" RAILS_SERVE_STATIC_FILES: "1" + WEB_CONCURRENCY: "2" + RAILS_MAX_THREADS: "5" + RAILS_DB_POOL: "5" + SIDEKIQ_CONCURRENCY: "5" command: - bundle - exec - rails - s - - -p - - "3000" - -b - 0.0.0.0 variables: diff --git a/Dockerfile b/Dockerfile index 7f0d26b..e7c10d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,7 +81,7 @@ COPY config/environments/production.rb /app/config/environments/beta.rb EXPOSE 3000 -CMD ["sh", "-lc", "bundle exec rails db:migrate && bundle exec rails s -p 3000 -b 0.0.0.0"] +CMD ["sh", "-lc", "bundle exec rails db:migrate && bundle exec rails s -b 0.0.0.0"] FROM runtime-base AS test diff --git a/config/database.yml b/config/database.yml index 2e10203..f339e82 100644 --- a/config/database.yml +++ b/config/database.yml @@ -6,7 +6,7 @@ # default: &default adapter: sqlite3 - pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + pool: <%= ENV.fetch("RAILS_DB_POOL", ENV.fetch("RAILS_MAX_THREADS", 5)) %> timeout: 5000 development: @@ -21,10 +21,12 @@ test: database: db/test.sqlite3 production: &production - url: <%= ENV.fetch("POSTGRES_URL", nil) %> + adapter: postgresql + encoding: unicode + url: <%= ENV["DATABASE_URL"] || ENV["POSTGRES_URL"] %> username: <%= ENV.fetch("POSTGRES_USERNAME", nil) %> password: <%= ENV.fetch("POSTGRES_PASSWORD", nil) %> - pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + pool: <%= ENV.fetch("RAILS_DB_POOL", ENV.fetch("RAILS_MAX_THREADS", 5)) %> beta: # just use the production settings diff --git a/config/puma.rb b/config/puma.rb index d9a94f3..661e38b 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -6,16 +6,17 @@ # the maximum value specified for Puma. Default is set to 5 threads for minimum # and maximum; this matches the default thread size of Active Record. # -threads_count = ENV.fetch('RAILS_MAX_THREADS', 5) -threads threads_count, threads_count +max_threads_count = ENV.fetch('RAILS_MAX_THREADS', 5) +min_threads_count = ENV.fetch('RAILS_MIN_THREADS', max_threads_count) +threads min_threads_count, max_threads_count # Specifies the `port` that Puma will listen on to receive requests; default is 3000. # -port ENV.fetch('PORT', 3000) +port ENV.fetch('PORT', 3000) # Specifies the `environment` that Puma will run in. # -environment ENV.fetch('RAILS_ENV', 'development') +environment ENV.fetch('RAILS_ENV', 'production') # Specifies the number of `workers` to boot in clustered mode. # Workers are forked webserver processes. If using threads and workers together @@ -23,14 +24,14 @@ environment ENV.fetch('RAILS_ENV', 'development') # Workers do not work on JRuby or Windows (both of which do not support # processes). # -# workers ENV.fetch("WEB_CONCURRENCY") { 2 } +workers ENV.fetch('WEB_CONCURRENCY', 2) # Use the `preload_app!` method when specifying a `workers` number. # This directive tells Puma to first boot the application and load code # before forking the application. This takes advantage of Copy On Write # process behavior so workers use less memory. # -# preload_app! +preload_app! # Allow puma to be restarted by `rails restart` command. plugin :tmp_restart diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 0000000..32d3b5d --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,2 @@ +--- +:concurrency: <%= ENV.fetch("SIDEKIQ_CONCURRENCY", 5) %> diff --git a/docker-compose.blackbox-follow.yml b/docker-compose.blackbox-follow.yml index a192dc1..25e946c 100644 --- a/docker-compose.blackbox-follow.yml +++ b/docker-compose.blackbox-follow.yml @@ -37,6 +37,12 @@ services: source-postgres: condition: service_healthy environment: + RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production} + WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-2} + RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5} + RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5} + SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5} + DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@source-postgres:5432/${TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB:-turniere_source_blackbox} POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@source-postgres:5432/${TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB:-turniere_source_blackbox} POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere} POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere} @@ -59,6 +65,12 @@ services: follower-postgres: condition: service_healthy environment: + RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production} + WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-2} + RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5} + RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5} + SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5} + DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@follower-postgres:5432/${TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB:-turniere_follower_blackbox} POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@follower-postgres:5432/${TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB:-turniere_follower_blackbox} POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere} POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere} diff --git a/docker-compose.blackbox.yml b/docker-compose.blackbox.yml index 15e5d3b..28a2244 100644 --- a/docker-compose.blackbox.yml +++ b/docker-compose.blackbox.yml @@ -19,6 +19,12 @@ services: postgres: condition: service_healthy environment: + RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production} + WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-2} + RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5} + RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5} + SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5} + DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@postgres:5432/${TURNIERE_BLACKBOX_POSTGRES_DB:-turniere_blackbox} POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@postgres:5432/${TURNIERE_BLACKBOX_POSTGRES_DB:-turniere_blackbox} POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere} POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere} diff --git a/spec/config/scaling_env_spec.rb b/spec/config/scaling_env_spec.rb new file mode 100644 index 0000000..791dd0c --- /dev/null +++ b/spec/config/scaling_env_spec.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require 'erb' +require 'yaml' + +RSpec.describe 'runtime scaling configuration' do + around do |example| + with_env( + 'DATABASE_URL' => nil, + 'POSTGRES_URL' => nil, + 'PORT' => nil, + 'RAILS_DB_POOL' => nil, + 'RAILS_ENV' => nil, + 'RAILS_MAX_THREADS' => nil, + 'RAILS_MIN_THREADS' => nil, + 'SIDEKIQ_CONCURRENCY' => nil, + 'WEB_CONCURRENCY' => nil + ) { example.run } + end + + it 'configures Puma workers and threads from environment variables' do + with_env( + 'PORT' => '4001', + 'RAILS_ENV' => 'production', + 'RAILS_MAX_THREADS' => '7', + 'RAILS_MIN_THREADS' => '2', + 'WEB_CONCURRENCY' => '4' + ) do + config = evaluate_puma_config + + expect(config.fetch(:threads)).to eq(%w[2 7]) + expect(config.fetch(:workers)).to eq('4') + expect(config.fetch(:preload_app)).to be(true) + expect(config.fetch(:port)).to eq('4001') + expect(config.fetch(:environment)).to eq('production') + expect(config.fetch(:plugin)).to eq(:tmp_restart) + end + end + + it 'uses safe Puma defaults' do + config = evaluate_puma_config + + expect(config.fetch(:threads)).to eq([5, 5]) + expect(config.fetch(:workers)).to eq(2) + expect(config.fetch(:port)).to eq(3000) + expect(config.fetch(:environment)).to eq('production') + end + + it 'configures the production database pool from RAILS_DB_POOL' do + with_env( + 'DATABASE_URL' => 'postgres://turniere:secret@postgres/turniere', + 'RAILS_DB_POOL' => '9', + 'RAILS_MAX_THREADS' => '7' + ) do + config = database_config.fetch('production') + + expect(config.fetch('adapter')).to eq('postgresql') + expect(config.fetch('encoding')).to eq('unicode') + expect(config.fetch('url')).to eq('postgres://turniere:secret@postgres/turniere') + expect(config.fetch('pool')).to eq(9) + end + end + + it 'falls back to RAILS_MAX_THREADS for the database pool and POSTGRES_URL for legacy deployments' do + with_env( + 'POSTGRES_URL' => 'postgres://turniere:secret@legacy-postgres/turniere', + 'RAILS_MAX_THREADS' => '6' + ) do + config = database_config.fetch('production') + + expect(config.fetch('url')).to eq('postgres://turniere:secret@legacy-postgres/turniere') + expect(config.fetch('pool')).to eq(6) + end + end + + it 'configures Sidekiq concurrency from SIDEKIQ_CONCURRENCY' do + with_env('SIDEKIQ_CONCURRENCY' => '10') do + expect(sidekiq_config).to include(':concurrency: 10') + end + + expect(sidekiq_config).to include(':concurrency: 5') + end + + def evaluate_puma_config + probe = puma_config_probe_class.new + probe.instance_eval(File.read(File.join(root, 'config/puma.rb')), File.join(root, 'config/puma.rb')) + probe.config + end + + def database_config + YAML.safe_load(ERB.new(File.read(File.join(root, 'config/database.yml'))).result, aliases: true) + end + + def sidekiq_config + ERB.new(File.read(File.join(root, 'config/sidekiq.yml'))).result + end + + def with_env(values) + previous = values.to_h { |key, _value| [key, ENV.fetch(key, nil)] } + values.each do |key, value| + value.nil? ? ENV.delete(key) : ENV[key] = value + end + + yield + ensure + previous.each do |key, value| + value.nil? ? ENV.delete(key) : ENV[key] = value + end + end + + def root + File.expand_path('../..', __dir__) + end + + def puma_config_probe_class + Class.new do + attr_reader :config + + def initialize + @config = {} + end + + def threads(min_threads, max_threads) + @config[:threads] = [min_threads, max_threads] + end + + def workers(count) + @config[:workers] = count + end + + def preload_app! + @config[:preload_app] = true + end + + def port(value) + @config[:port] = value + end + + def environment(value) + @config[:environment] = value + end + + def plugin(name) + @config[:plugin] = name + end + end + end +end diff --git a/tasks.py b/tasks.py index c1f323a..2e5efe6 100644 --- a/tasks.py +++ b/tasks.py @@ -470,12 +470,19 @@ def _spec_shards(node_total): def _shared_production_env(): - return _env( - RAILS_ENV="production", - POSTGRES_URL=os.environ.get( + database_url = os.environ.get( + "DATABASE_URL", + os.environ.get( "POSTGRES_URL", f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@postgres:5432/{BLACKBOX_DB_NAME}", ), + ) + rails_max_threads = os.environ.get("RAILS_MAX_THREADS", "5") + + return _env( + RAILS_ENV="production", + DATABASE_URL=database_url, + POSTGRES_URL=database_url, POSTGRES_USERNAME=os.environ.get("POSTGRES_USERNAME", BLACKBOX_DB_USER), POSTGRES_PASSWORD=os.environ.get("POSTGRES_PASSWORD", BLACKBOX_DB_PASSWORD), SECRET_KEY_BASE=os.environ.get("SECRET_KEY_BASE", BLACKBOX_SECRET_KEY_BASE), @@ -484,13 +491,20 @@ def _shared_production_env(): TURNIERE_DISABLE_EMAIL_DELIVERY=os.environ.get("TURNIERE_DISABLE_EMAIL_DELIVERY", "1"), RAILS_LOG_TO_STDOUT=os.environ.get("RAILS_LOG_TO_STDOUT", "1"), RAILS_SERVE_STATIC_FILES=os.environ.get("RAILS_SERVE_STATIC_FILES", "1"), + WEB_CONCURRENCY=os.environ.get("WEB_CONCURRENCY", "2"), + RAILS_MAX_THREADS=rails_max_threads, + RAILS_DB_POOL=os.environ.get("RAILS_DB_POOL", rails_max_threads), + SIDEKIQ_CONCURRENCY=os.environ.get("SIDEKIQ_CONCURRENCY", "5"), ) def _production_env_for(db_host, db_name): + database_url = f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@{db_host}:5432/{db_name}" + return _env( RAILS_ENV="production", - POSTGRES_URL=f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@{db_host}:5432/{db_name}", + DATABASE_URL=database_url, + POSTGRES_URL=database_url, POSTGRES_USERNAME=BLACKBOX_DB_USER, POSTGRES_PASSWORD=BLACKBOX_DB_PASSWORD, SECRET_KEY_BASE=BLACKBOX_SECRET_KEY_BASE, @@ -499,6 +513,10 @@ def _production_env_for(db_host, db_name): TURNIERE_DISABLE_EMAIL_DELIVERY="1", RAILS_LOG_TO_STDOUT="1", RAILS_SERVE_STATIC_FILES="1", + WEB_CONCURRENCY=os.environ.get("WEB_CONCURRENCY", "2"), + RAILS_MAX_THREADS=os.environ.get("RAILS_MAX_THREADS", "5"), + RAILS_DB_POOL=os.environ.get("RAILS_DB_POOL", os.environ.get("RAILS_MAX_THREADS", "5")), + SIDEKIQ_CONCURRENCY=os.environ.get("SIDEKIQ_CONCURRENCY", "5"), ) @@ -1094,6 +1112,101 @@ def docker_build_all(c): docker_build_test(c) +def _assert_output_match(label, output, pattern): + if re.search(pattern, output) is not None: + return + + tail = output[-4000:] if output else "" + raise RuntimeError(f"{label} did not match {pattern!r}.\nLast output:\n{tail}") + + +def _blackbox_app_exec(env, *command): + result = _compose_run(["exec", "-T", "app", *command], env=env, capture_output=True) + output = "\n".join(part for part in (result.stdout, result.stderr) if part) + return output.strip() + + +def _assert_blackbox_scaling_config(env, expected): + logs = _compose_run(["logs", "app"], env=env, capture_output=True).stdout + + _assert_output_match("Puma worker count", logs, rf"Workers:\s+{re.escape(expected['web_concurrency'])}\b") + _assert_output_match("Puma max threads", logs, rf"Max threads:\s+{re.escape(expected['rails_max_threads'])}\b") + _assert_output_match("Puma environment", logs, r"Environment:\s+production\b") + + pool_size = _blackbox_app_exec(env, "bundle", "exec", "rails", "runner", "puts ActiveRecord::Base.connection_pool.size") + if pool_size.splitlines()[-1] != expected["rails_db_pool"]: + raise RuntimeError(f"Expected DB pool {expected['rails_db_pool']}, got {pool_size!r}") + + sidekiq_config = _blackbox_app_exec( + env, + "ruby", + "-rerb", + "-e", + "puts ERB.new(File.read('config/sidekiq.yml')).result", + ) + expected_sidekiq = f":concurrency: {expected['sidekiq_concurrency']}" + if expected_sidekiq not in sidekiq_config: + raise RuntimeError(f"Expected Sidekiq config {expected_sidekiq!r}, got {sidekiq_config!r}") + + +@task(name="docker-blackbox-scaling-config") +def docker_blackbox_scaling_config( + c, + host_port="3004", + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, + build=True, +): + """Build once, boot the production image with two scaling env sets, and assert runtime config.""" + if build: + docker_build_production(c, tag=app_image) + + cases = [ + { + "label": "medium", + "web_concurrency": "2", + "rails_max_threads": "3", + "rails_db_pool": "4", + "sidekiq_concurrency": "6", + }, + { + "label": "event", + "web_concurrency": "3", + "rails_max_threads": "4", + "rails_db_pool": "5", + "sidekiq_concurrency": "7", + }, + ] + + for expected in cases: + env = _compose_env( + host_port=host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + expect_profiling="false", + ) + env.update( + TURNIERE_BLACKBOX_RAILS_ENV="production", + TURNIERE_BLACKBOX_WEB_CONCURRENCY=expected["web_concurrency"], + TURNIERE_BLACKBOX_RAILS_MAX_THREADS=expected["rails_max_threads"], + TURNIERE_BLACKBOX_RAILS_DB_POOL=expected["rails_db_pool"], + TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY=expected["sidekiq_concurrency"], + ) + + _print_header(f"Scaling config case: {expected['label']}") + try: + _compose_run(["down", "-v", "--remove-orphans"], env=env, check=False) + _compose_run(["up", "-d", "postgres", "app"], env=env) + _wait_for_http(base_url=f"http://127.0.0.1:{host_port}", timeout=120) + _assert_blackbox_scaling_config(env, expected) + finally: + _compose_run(["down", "-v", "--remove-orphans"], env=env, check=False) + + print("Scaling config changed across boots without rebuilding the production image.") + + @task(name="docker-blackbox-up") def docker_blackbox_up( c,