turniere
/
turniere-backend
mirror of https://github.com/turniere/turniere-backend.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into codex/tur-84-sync-token

This commit is contained in:
Daniel Schädler 2026-04-30 09:33:01 +02:00
parent 2b63e61475 5ace19eeea
commit a95fabacf7
9 changed files with 318 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
.gitlab-ci.yml
View File

@ -57,13 +57,15 @@ rails e2e spec:
alias: app
variables:
TURNIERE_DISABLE_EMAIL_DELIVERY: "1"
WEB_CONCURRENCY: "1"
RAILS_MAX_THREADS: "5"
RAILS_DB_POOL: "5"
SIDEKIQ_CONCURRENCY: "5"
command:
- bundle
- exec
- rails
- s
- -p
- "3000"
- -b
- 0.0.0.0
variables:
@ -71,6 +73,7 @@ rails e2e spec:
POSTGRES_DB: turniere_blackbox
POSTGRES_USER: turniere
POSTGRES_PASSWORD: turniere
DATABASE_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
POSTGRES_USERNAME: turniere
SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod
@ -112,13 +115,15 @@ rails load smoke:
alias: app
variables:
TURNIERE_DISABLE_EMAIL_DELIVERY: "1"
WEB_CONCURRENCY: "1"
RAILS_MAX_THREADS: "5"
RAILS_DB_POOL: "5"
SIDEKIQ_CONCURRENCY: "5"
command:
- bundle
- exec
- rails
- s
- -p
- "3000"
- -b
- 0.0.0.0
variables:
@ -126,6 +131,7 @@ rails load smoke:
POSTGRES_DB: turniere_blackbox
POSTGRES_USER: turniere
POSTGRES_PASSWORD: turniere
DATABASE_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
POSTGRES_USERNAME: turniere
SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod
@ -181,6 +187,7 @@ rails follow sync e2e spec:
- name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA
alias: source-app
variables:
DATABASE_URL: postgres://turniere:turniere@source-postgres:5432/turniere_source_blackbox
POSTGRES_URL: postgres://turniere:turniere@source-postgres:5432/turniere_source_blackbox
POSTGRES_USERNAME: turniere
POSTGRES_PASSWORD: turniere
@ -193,18 +200,21 @@ rails follow sync e2e spec:
no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1
RAILS_LOG_TO_STDOUT: "1"
RAILS_SERVE_STATIC_FILES: "1"
WEB_CONCURRENCY: "1"
RAILS_MAX_THREADS: "5"
RAILS_DB_POOL: "5"
SIDEKIQ_CONCURRENCY: "5"
command:
- bundle
- exec
- rails
- s
- -p
- "3000"
- -b
- 0.0.0.0
- name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA
alias: follower-app
variables:
DATABASE_URL: postgres://turniere:turniere@follower-postgres:5432/turniere_follower_blackbox
POSTGRES_URL: postgres://turniere:turniere@follower-postgres:5432/turniere_follower_blackbox
POSTGRES_USERNAME: turniere
POSTGRES_PASSWORD: turniere
@ -217,13 +227,15 @@ rails follow sync e2e spec:
no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1
RAILS_LOG_TO_STDOUT: "1"
RAILS_SERVE_STATIC_FILES: "1"
WEB_CONCURRENCY: "1"
RAILS_MAX_THREADS: "5"
RAILS_DB_POOL: "5"
SIDEKIQ_CONCURRENCY: "5"
command:
- bundle
- exec
- rails
- s
- -p
- "3000"
- -b
- 0.0.0.0
variables:

2
Dockerfile
View File

@ -81,7 +81,7 @@ COPY config/environments/production.rb /app/config/environments/beta.rb
EXPOSE 3000
CMD ["sh", "-lc", "bundle exec rails db:migrate && bundle exec rails s -p 3000 -b 0.0.0.0"]
CMD ["sh", "-lc", "bundle exec rails db:migrate && bundle exec rails s -b 0.0.0.0"]
FROM runtime-base AS test

8
config/database.yml
View File

@ -6,7 +6,7 @@
#
default: &default
adapter: sqlite3
pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
pool: <%= ENV.fetch("RAILS_DB_POOL", ENV.fetch("RAILS_MAX_THREADS", 5)) %>
timeout: 5000
development:
@ -21,10 +21,12 @@ test:
database: db/test.sqlite3
production: &production
url: <%= ENV.fetch("POSTGRES_URL", nil) %>
adapter: postgresql
encoding: unicode
url: <%= ENV["DATABASE_URL"] || ENV["POSTGRES_URL"] %>
username: <%= ENV.fetch("POSTGRES_USERNAME", nil) %>
password: <%= ENV.fetch("POSTGRES_PASSWORD", nil) %>
pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
pool: <%= ENV.fetch("RAILS_DB_POOL", ENV.fetch("RAILS_MAX_THREADS", 5)) %>
beta:
# just use the production settings

13
config/puma.rb
View File

@ -6,16 +6,17 @@
# the maximum value specified for Puma. Default is set to 5 threads for minimum
# and maximum; this matches the default thread size of Active Record.
#
threads_count = ENV.fetch('RAILS_MAX_THREADS', 5)
threads threads_count, threads_count
max_threads_count = ENV.fetch('RAILS_MAX_THREADS', 5)
min_threads_count = ENV.fetch('RAILS_MIN_THREADS', max_threads_count)
threads min_threads_count, max_threads_count
# Specifies the `port` that Puma will listen on to receive requests; default is 3000.
#
port ENV.fetch('PORT', 3000)
port ENV.fetch('PORT', 3000)
# Specifies the `environment` that Puma will run in.
#
environment ENV.fetch('RAILS_ENV', 'development')
environment ENV.fetch('RAILS_ENV', 'production')
# Specifies the number of `workers` to boot in clustered mode.
# Workers are forked webserver processes. If using threads and workers together
@ -23,14 +24,14 @@ environment ENV.fetch('RAILS_ENV', 'development')
# Workers do not work on JRuby or Windows (both of which do not support
# processes).
#
# workers ENV.fetch("WEB_CONCURRENCY") { 2 }
workers ENV.fetch('WEB_CONCURRENCY', 2)
# Use the `preload_app!` method when specifying a `workers` number.
# This directive tells Puma to first boot the application and load code
# before forking the application. This takes advantage of Copy On Write
# process behavior so workers use less memory.
#
# preload_app!
preload_app!
# Allow puma to be restarted by `rails restart` command.
plugin :tmp_restart

2
config/sidekiq.yml Normal file
View File

@ -0,0 +1,2 @@
---
:concurrency: <%= ENV.fetch("SIDEKIQ_CONCURRENCY", 5) %>

12
docker-compose.blackbox-follow.yml
View File

@ -37,6 +37,12 @@ services:
source-postgres:
condition: service_healthy
environment:
RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production}
WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-1}
RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5}
RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5}
SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5}
DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@source-postgres:5432/${TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB:-turniere_source_blackbox}
POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@source-postgres:5432/${TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB:-turniere_source_blackbox}
POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}
POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}
@ -59,6 +65,12 @@ services:
follower-postgres:
condition: service_healthy
environment:
RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production}
WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-1}
RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5}
RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5}
SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5}
DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@follower-postgres:5432/${TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB:-turniere_follower_blackbox}
POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@follower-postgres:5432/${TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB:-turniere_follower_blackbox}
POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}
POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}

6
docker-compose.blackbox.yml
View File

@ -19,6 +19,12 @@ services:
postgres:
condition: service_healthy
environment:
RAILS_ENV: ${TURNIERE_BLACKBOX_RAILS_ENV:-production}
WEB_CONCURRENCY: ${TURNIERE_BLACKBOX_WEB_CONCURRENCY:-1}
RAILS_MAX_THREADS: ${TURNIERE_BLACKBOX_RAILS_MAX_THREADS:-5}
RAILS_DB_POOL: ${TURNIERE_BLACKBOX_RAILS_DB_POOL:-5}
SIDEKIQ_CONCURRENCY: ${TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY:-5}
DATABASE_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@postgres:5432/${TURNIERE_BLACKBOX_POSTGRES_DB:-turniere_blackbox}
POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@postgres:5432/${TURNIERE_BLACKBOX_POSTGRES_DB:-turniere_blackbox}
POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}
POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}

148
spec/config/scaling_env_spec.rb Normal file
View File

@ -0,0 +1,148 @@
# frozen_string_literal: true
require 'erb'
require 'yaml'
RSpec.describe 'runtime scaling configuration' do
around do |example|
with_env(
'DATABASE_URL' => nil,
'POSTGRES_URL' => nil,
'PORT' => nil,
'RAILS_DB_POOL' => nil,
'RAILS_ENV' => nil,
'RAILS_MAX_THREADS' => nil,
'RAILS_MIN_THREADS' => nil,
'SIDEKIQ_CONCURRENCY' => nil,
'WEB_CONCURRENCY' => nil
) { example.run }
end
it 'configures Puma workers and threads from environment variables' do
with_env(
'PORT' => '4001',
'RAILS_ENV' => 'production',
'RAILS_MAX_THREADS' => '7',
'RAILS_MIN_THREADS' => '2',
'WEB_CONCURRENCY' => '4'
) do
config = evaluate_puma_config
expect(config.fetch(:threads)).to eq(%w[2 7])
expect(config.fetch(:workers)).to eq('4')
expect(config.fetch(:preload_app)).to be(true)
expect(config.fetch(:port)).to eq('4001')
expect(config.fetch(:environment)).to eq('production')
expect(config.fetch(:plugin)).to eq(:tmp_restart)
end
end
it 'uses safe Puma defaults' do
config = evaluate_puma_config
expect(config.fetch(:threads)).to eq([5, 5])
expect(config.fetch(:workers)).to eq(2)
expect(config.fetch(:port)).to eq(3000)
expect(config.fetch(:environment)).to eq('production')
end
it 'configures the production database pool from RAILS_DB_POOL' do
with_env(
'DATABASE_URL' => 'postgres://turniere:secret@postgres/turniere',
'RAILS_DB_POOL' => '9',
'RAILS_MAX_THREADS' => '7'
) do
config = database_config.fetch('production')
expect(config.fetch('adapter')).to eq('postgresql')
expect(config.fetch('encoding')).to eq('unicode')
expect(config.fetch('url')).to eq('postgres://turniere:secret@postgres/turniere')
expect(config.fetch('pool')).to eq(9)
end
end
it 'falls back to RAILS_MAX_THREADS for the database pool and POSTGRES_URL for legacy deployments' do
with_env(
'POSTGRES_URL' => 'postgres://turniere:secret@legacy-postgres/turniere',
'RAILS_MAX_THREADS' => '6'
) do
config = database_config.fetch('production')
expect(config.fetch('url')).to eq('postgres://turniere:secret@legacy-postgres/turniere')
expect(config.fetch('pool')).to eq(6)
end
end
it 'configures Sidekiq concurrency from SIDEKIQ_CONCURRENCY' do
with_env('SIDEKIQ_CONCURRENCY' => '10') do
expect(sidekiq_config).to include(':concurrency: 10')
end
expect(sidekiq_config).to include(':concurrency: 5')
end
def evaluate_puma_config
probe = puma_config_probe_class.new
probe.instance_eval(File.read(File.join(root, 'config/puma.rb')), File.join(root, 'config/puma.rb'))
probe.config
end
def database_config
YAML.safe_load(ERB.new(File.read(File.join(root, 'config/database.yml'))).result, aliases: true)
end
def sidekiq_config
ERB.new(File.read(File.join(root, 'config/sidekiq.yml'))).result
end
def with_env(values)
previous = values.to_h { |key, _value| [key, ENV.fetch(key, nil)] }
values.each do |key, value|
value.nil? ? ENV.delete(key) : ENV[key] = value
end
yield
ensure
previous.each do |key, value|
value.nil? ? ENV.delete(key) : ENV[key] = value
end
end
def root
File.expand_path('../..', __dir__)
end
def puma_config_probe_class
Class.new do
attr_reader :config
def initialize
@config = {}
end
def threads(min_threads, max_threads)
@config[:threads] = [min_threads, max_threads]
end
def workers(count)
@config[:workers] = count
end
def preload_app!
@config[:preload_app] = true
end
def port(value)
@config[:port] = value
end
def environment(value)
@config[:environment] = value
end
def plugin(name)
@config[:plugin] = name
end
end
end
end

121
tasks.py
View File

@ -470,12 +470,19 @@ def _spec_shards(node_total):
def _shared_production_env():
return _env(
RAILS_ENV="production",
POSTGRES_URL=os.environ.get(
database_url = os.environ.get(
"DATABASE_URL",
os.environ.get(
"POSTGRES_URL",
f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@postgres:5432/{BLACKBOX_DB_NAME}",
),
)
rails_max_threads = os.environ.get("RAILS_MAX_THREADS", "5")
return _env(
RAILS_ENV="production",
DATABASE_URL=database_url,
POSTGRES_URL=database_url,
POSTGRES_USERNAME=os.environ.get("POSTGRES_USERNAME", BLACKBOX_DB_USER),
POSTGRES_PASSWORD=os.environ.get("POSTGRES_PASSWORD", BLACKBOX_DB_PASSWORD),
SECRET_KEY_BASE=os.environ.get("SECRET_KEY_BASE", BLACKBOX_SECRET_KEY_BASE),
@ -484,13 +491,20 @@ def _shared_production_env():
TURNIERE_DISABLE_EMAIL_DELIVERY=os.environ.get("TURNIERE_DISABLE_EMAIL_DELIVERY", "1"),
RAILS_LOG_TO_STDOUT=os.environ.get("RAILS_LOG_TO_STDOUT", "1"),
RAILS_SERVE_STATIC_FILES=os.environ.get("RAILS_SERVE_STATIC_FILES", "1"),
WEB_CONCURRENCY=os.environ.get("WEB_CONCURRENCY", "2"),
RAILS_MAX_THREADS=rails_max_threads,
RAILS_DB_POOL=os.environ.get("RAILS_DB_POOL", rails_max_threads),
SIDEKIQ_CONCURRENCY=os.environ.get("SIDEKIQ_CONCURRENCY", "5"),
)
def _production_env_for(db_host, db_name):
database_url = f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@{db_host}:5432/{db_name}"
return _env(
RAILS_ENV="production",
POSTGRES_URL=f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@{db_host}:5432/{db_name}",
DATABASE_URL=database_url,
POSTGRES_URL=database_url,
POSTGRES_USERNAME=BLACKBOX_DB_USER,
POSTGRES_PASSWORD=BLACKBOX_DB_PASSWORD,
SECRET_KEY_BASE=BLACKBOX_SECRET_KEY_BASE,
@ -499,6 +513,10 @@ def _production_env_for(db_host, db_name):
TURNIERE_DISABLE_EMAIL_DELIVERY="1",
RAILS_LOG_TO_STDOUT="1",
RAILS_SERVE_STATIC_FILES="1",
WEB_CONCURRENCY=os.environ.get("WEB_CONCURRENCY", "2"),
RAILS_MAX_THREADS=os.environ.get("RAILS_MAX_THREADS", "5"),
RAILS_DB_POOL=os.environ.get("RAILS_DB_POOL", os.environ.get("RAILS_MAX_THREADS", "5")),
SIDEKIQ_CONCURRENCY=os.environ.get("SIDEKIQ_CONCURRENCY", "5"),
)
@ -1094,6 +1112,101 @@ def docker_build_all(c):
docker_build_test(c)
def _assert_output_match(label, output, pattern):
if re.search(pattern, output) is not None:
return
tail = output[-4000:] if output else "<empty output>"
raise RuntimeError(f"{label} did not match {pattern!r}.\nLast output:\n{tail}")
def _blackbox_app_exec(env, *command):
result = _compose_run(["exec", "-T", "app", *command], env=env, capture_output=True)
output = "\n".join(part for part in (result.stdout, result.stderr) if part)
return output.strip()
def _assert_blackbox_scaling_config(env, expected):
logs = _compose_run(["logs", "app"], env=env, capture_output=True).stdout
_assert_output_match("Puma worker count", logs, rf"Workers:\s+{re.escape(expected['web_concurrency'])}\b")
_assert_output_match("Puma max threads", logs, rf"Max threads:\s+{re.escape(expected['rails_max_threads'])}\b")
_assert_output_match("Puma environment", logs, r"Environment:\s+production\b")
pool_size = _blackbox_app_exec(env, "bundle", "exec", "rails", "runner", "puts ActiveRecord::Base.connection_pool.size")
if pool_size.splitlines()[-1] != expected["rails_db_pool"]:
raise RuntimeError(f"Expected DB pool {expected['rails_db_pool']}, got {pool_size!r}")
sidekiq_config = _blackbox_app_exec(
env,
"ruby",
"-rerb",
"-e",
"puts ERB.new(File.read('config/sidekiq.yml')).result",
)
expected_sidekiq = f":concurrency: {expected['sidekiq_concurrency']}"
if expected_sidekiq not in sidekiq_config:
raise RuntimeError(f"Expected Sidekiq config {expected_sidekiq!r}, got {sidekiq_config!r}")
@task(name="docker-blackbox-scaling-config")
def docker_blackbox_scaling_config(
c,
host_port="3004",
app_image=PRODUCTION_TAG,
runner_image=TEST_TAG,
postgres_image=BLACKBOX_POSTGRES_IMAGE,
build=True,
):
"""Build once, boot the production image with two scaling env sets, and assert runtime config."""
if build:
docker_build_production(c, tag=app_image)
cases = [
{
"label": "medium",
"web_concurrency": "2",
"rails_max_threads": "3",
"rails_db_pool": "4",
"sidekiq_concurrency": "6",
},
{
"label": "event",
"web_concurrency": "3",
"rails_max_threads": "4",
"rails_db_pool": "5",
"sidekiq_concurrency": "7",
},
]
for expected in cases:
env = _compose_env(
host_port=host_port,
app_image=app_image,
runner_image=runner_image,
postgres_image=postgres_image,
expect_profiling="false",
)
env.update(
TURNIERE_BLACKBOX_RAILS_ENV="production",
TURNIERE_BLACKBOX_WEB_CONCURRENCY=expected["web_concurrency"],
TURNIERE_BLACKBOX_RAILS_MAX_THREADS=expected["rails_max_threads"],
TURNIERE_BLACKBOX_RAILS_DB_POOL=expected["rails_db_pool"],
TURNIERE_BLACKBOX_SIDEKIQ_CONCURRENCY=expected["sidekiq_concurrency"],
)
_print_header(f"Scaling config case: {expected['label']}")
try:
_compose_run(["down", "-v", "--remove-orphans"], env=env, check=False)
_compose_run(["up", "-d", "postgres", "app"], env=env)
_wait_for_http(base_url=f"http://127.0.0.1:{host_port}", timeout=120)
_assert_blackbox_scaling_config(env, expected)
finally:
_compose_run(["down", "-v", "--remove-orphans"], env=env, check=False)
print("Scaling config changed across boots without rebuilding the production image.")
@task(name="docker-blackbox-up")
def docker_blackbox_up(
c,