feat(load): add load smoke tests

.gitlab-ci.yml

@@ -98,6 +98,66 @@ rails e2e spec:
     - cd /app
     - inv blackbox-service
 
+rails load smoke:
+  stage: e2e
+  tags:
+    - docker
+  needs:
+    - job: build_image
+  image: $CI_REGISTRY_IMAGE/test/commits:$CI_COMMIT_SHA
+  services:
+    - name: postgres:16
+      alias: postgres
+    - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA
+      alias: app
+      variables:
+        TURNIERE_DISABLE_EMAIL_DELIVERY: "1"
+      command:
+        - bundle
+        - exec
+        - rails
+        - s
+        - -p
+        - "3000"
+        - -b
+        - 0.0.0.0
+  variables:
+    FF_NETWORK_PER_BUILD: "true"
+    POSTGRES_DB: turniere_blackbox
+    POSTGRES_USER: turniere
+    POSTGRES_PASSWORD: turniere
+    POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox
+    POSTGRES_USERNAME: turniere
+    SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod
+    MAILGUN_API_KEY: blackbox-test-api-key
+    MAILGUN_DOMAIN: blackbox.example.com
+    RAILS_LOG_TO_STDOUT: "1"
+    RAILS_SERVE_STATIC_FILES: "1"
+    NO_PROXY: app,postgres,localhost,127.0.0.1
+    no_proxy: app,postgres,localhost,127.0.0.1
+    TURNIERE_E2E_BASE_URL: http://app:3000
+    TURNIERE_E2E_EMAIL: e2e@example.com
+    TURNIERE_E2E_PASSWORD: password123
+    TURNIERE_E2E_USERNAME: e2e-user
+    TURNIERE_LOAD_HTTP_CLIENTS: "1,10,50,100"
+    TURNIERE_LOAD_WEBSOCKET_CLIENTS: "1,10,50,100,200"
+    TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT: "3"
+    TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS: "1.0"
+    TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE: "25"
+  rules:
+    - if: $SKIP_TEST
+      when: never
+    - if: $SKIP_ALL_SPECS
+      when: never
+    - if: $SKIP_LOAD_SPEC
+      when: never
+    - if: $CI_PIPELINE_SOURCE != "push"
+      when: never
+    - when: always
+  script:
+    - cd /app
+    - inv blackbox-load-service
+
 rails follow sync e2e spec:
   stage: e2e
   tags:

doc/load_tests.md

@@ -0,0 +1,39 @@
+# Load Tests
+
+Load smoke tests live in `e2e/load_test.rb` and run through invoke tasks.
+
+The runner creates a realistic group-stage tournament over HTTP, then measures:
+
+- concurrent `GET /tournaments/:id` reads at each configured client level
+- active ActionCable subscriptions to `TournamentChannel`, opened in bounded
+  batches so the test measures 200 active sockets instead of only handshake
+  burst capacity
+- `/metrics` websocket gauges and process counters after the run
+
+It prints relative timings such as p50, p95, max, and requests per second. The
+CI goal is not a fixed benchmark. The job fails when requests error, websocket
+clients cannot subscribe, or ActionCable connection metrics do not reflect the
+opened clients.
+
+Useful commands:
+
+- `inv verify-load-test`
+- `inv load-test --base-url=http://127.0.0.1:3000`
+- `inv load-test --http-clients=1,10,50,100 --websocket-clients=1,10,50,100,200`
+- `inv blackbox-load-service`
+
+Useful environment overrides:
+
+- `TURNIERE_LOAD_HTTP_CLIENTS=1,10,50,100`
+- `TURNIERE_LOAD_WEBSOCKET_CLIENTS=1,10,50,100,200`
+- `TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT=3`
+- `TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS=1.0`
+- `TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE=25`
+- `TURNIERE_LOAD_GROUP_COUNT=4`
+- `TURNIERE_LOAD_TEAMS_PER_GROUP=4`
+- `TURNIERE_LOAD_PLAYOFF_TEAMS=8`
+- `TURNIERE_LOAD_FORMAT=json`
+
+GitLab CI runs `rails load smoke` against the production image with Postgres,
+covering read pressure up to 100 HTTP clients and active websocket pressure up
+to 200 clients.

e2e/lib/load_test_runner.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+
+require_relative 'action_cable_client'
+require_relative 'api_client'
+require_relative 'scenario_runner'
+
+module TurniereE2E
+  class LoadTestRunner
+    DEFAULT_HTTP_CLIENTS = [1, 10, 50, 100].freeze
+    DEFAULT_WEBSOCKET_CLIENTS = [1, 10, 50, 100, 200].freeze
+
+    def initialize(**options)
+      @base_url = options.fetch(:base_url).delete_suffix('/')
+      @email = options[:email]
+      @password = options[:password]
+      @username = options[:username]
+      @tournament_id = options[:tournament_id]
+      @http_clients = options.fetch(:http_clients, DEFAULT_HTTP_CLIENTS)
+      @websocket_clients = options.fetch(:websocket_clients, DEFAULT_WEBSOCKET_CLIENTS)
+      @http_requests_per_client = options.fetch(:http_requests_per_client, 3).to_i
+      @websocket_hold_seconds = options.fetch(:websocket_hold_seconds, 1.0).to_f
+      @websocket_connect_batch_size = options.fetch(:websocket_connect_batch_size, 25).to_i
+      @group_count = options.fetch(:group_count, 4).to_i
+      @teams_per_group = options.fetch(:teams_per_group, 4).to_i
+      @playoff_teams_amount = options.fetch(:playoff_teams_amount, 8).to_i
+      @fail_on_error = options.fetch(:fail_on_error, true)
+    end
+
+    def run
+      tournament = prepare_tournament
+      result = {
+        base_url: base_url,
+        tournament: tournament,
+        http: http_clients.map { |client_count| run_http_level(tournament, client_count) },
+        websocket: websocket_clients.map { |client_count| run_websocket_level(tournament, client_count) },
+        metrics: metrics_summary
+      }
+
+      assert_success!(result) if fail_on_error
+      result
+    end
+
+    private
+
+    attr_reader :base_url, :email, :password, :username, :tournament_id, :http_clients, :websocket_clients,
+                :http_requests_per_client, :websocket_hold_seconds, :websocket_connect_batch_size, :group_count,
+                :teams_per_group, :playoff_teams_amount, :fail_on_error
+
+    def prepare_tournament
+      return fetch_existing_tournament if tournament_id.to_s.strip != ''
+
+      runner = ScenarioRunner.new(base_url: base_url, email: email, password: password, username: username)
+      runner.wait_for_healthcheck!
+      result = runner.run_group_stage_render_profile(
+        group_count: group_count,
+        teams_per_group: teams_per_group,
+        playoff_teams_amount: playoff_teams_amount,
+        stop_at: :created
+      )
+      result.fetch(:tournament)
+    end
+
+    def fetch_existing_tournament
+      client = ApiClient.new(base_url: base_url)
+      response = client.get("/tournaments/#{tournament_id}")
+      unless response.fetch(:status) == 200
+        raise ApiError.new(
+          "load-test tournament fetch failed with #{response.fetch(:status)}",
+          status: response.fetch(:status),
+          body: response.fetch(:json)
+        )
+      end
+
+      summarize_tournament(response.fetch(:json))
+    end
+
+    def run_http_level(tournament, client_count)
+      path = "/tournaments/#{tournament.fetch(:id)}"
+      records = run_threads(client_count) do
+        client = ApiClient.new(base_url: base_url)
+        Array.new(http_requests_per_client) { measure_http_request(client, path) }
+      end.flatten
+
+      http_summary(client_count, records)
+    end
+
+    def run_websocket_level(tournament, client_count)
+      clients = []
+      baseline_connections = metric_scalar('turniere_action_cable_connections').to_i
+      records = run_websocket_open_batches(tournament.fetch(:id), client_count, clients)
+      successful = records.count { |record| record[:ok] }
+      observed_connections = wait_for_connection_metric(successful)
+      sleep websocket_hold_seconds if websocket_hold_seconds.positive?
+
+      websocket_summary(client_count, records, baseline_connections, successful, observed_connections)
+    ensure
+      clients.each(&:close)
+    end
+
+    def run_websocket_open_batches(tournament_id, client_count, clients)
+      batch_size = websocket_connect_batch_size.positive? ? websocket_connect_batch_size : client_count
+
+      Array.new(client_count) { true }.each_slice(batch_size).flat_map do |batch|
+        run_threads(batch.count) do
+          measure_websocket_open(tournament_id).tap do |record|
+            clients << record.delete(:client) if record[:client]
+          end
+        end
+      end
+    end
+
+    def run_threads(count)
+      return [] if count.to_i <= 0
+
+      ready = Queue.new
+      start = Queue.new
+      results = Queue.new
+      threads = Array.new(count) do
+        Thread.new do
+          ready << true
+          start.pop
+          results << yield
+        rescue StandardError => e
+          results << [{ error: "#{e.class}: #{e.message}" }]
+        end
+      end
+
+      count.times { ready.pop }
+      started_at = monotonic_time
+      count.times { start << true }
+      threads.each(&:join)
+      finished_at = monotonic_time
+
+      drain_queue(results).flatten.map do |record|
+        record.is_a?(Hash) ? record.merge(level_duration_seconds: finished_at - started_at) : record
+      end
+    end
+
+    def measure_http_request(client, path)
+      started_at = monotonic_time
+      response = client.get(path)
+      {
+        status: response.fetch(:status),
+        duration_ms: elapsed_ms(started_at),
+        body: error_body(response)
+      }.compact
+    rescue StandardError => e
+      { error: "#{e.class}: #{e.message}", duration_ms: elapsed_ms(started_at) }
+    end
+
+    def measure_websocket_open(tournament_id)
+      client = ActionCableClient.new(base_url: base_url)
+      started_at = monotonic_time
+      client.subscribe_tournament!(tournament_id: tournament_id)
+      payload = client.wait_for_tournament_payload!(timeout: 20)
+      {
+        ok: true,
+        duration_ms: elapsed_ms(started_at),
+        payload_type: payload.fetch('type'),
+        client: client
+      }
+    rescue StandardError => e
+      client&.close
+      { ok: false, duration_ms: elapsed_ms(started_at), error: "#{e.class}: #{e.message}" }
+    end
+
+    def wait_for_connection_metric(expected, timeout: 10)
+      deadline = monotonic_time + timeout
+      loop do
+        value = metric_scalar('turniere_action_cable_connections')
+        return value if value && value >= expected
+        return value if monotonic_time >= deadline
+
+        sleep 0.2
+      end
+    end
+
+    def metrics_summary
+      {
+        action_cable_connections: metric_scalar('turniere_action_cable_connections'),
+        action_cable_connections_total: metric_scalar('turniere_action_cable_connections_total'),
+        ruby_threads: metric_scalar('turniere_ruby_threads'),
+        active_record_connections: metric_samples('turniere_active_record_connection_pool')
+      }
+    end
+
+    def metric_scalar(name)
+      metric_samples(name).map { |sample| sample.fetch(:value) }.max
+    end
+
+    def metric_samples(name)
+      body = metrics_body
+      body.lines.filter_map do |line|
+        next if line.start_with?('#')
+
+        sample, value = line.split(/\s+/, 2)
+        next unless sample == name || sample.start_with?("#{name}{")
+
+        { sample: sample, value: value.to_f }
+      end
+    end
+
+    def metrics_body
+      response = ApiClient.new(base_url: base_url).get('/metrics')
+      return '' unless response.fetch(:status) == 200
+
+      response.dig(:json, :raw_body).to_s
+    end
+
+    def assert_success!(result)
+      failures = []
+      result.fetch(:http).each { |level| failures.concat(http_failures(level)) }
+      result.fetch(:websocket).each { |level| failures.concat(websocket_failures(level)) }
+
+      raise "load test failed: #{failures.join('; ')}" unless failures.empty?
+    end
+
+    def http_summary(client_count, records)
+      durations = records.filter_map { |record| record[:duration_ms] }
+      errors = failed_http_records(records)
+
+      {
+        clients: client_count,
+        requests: records.count,
+        ok: records.count - errors.count,
+        errors: errors.count,
+        statuses: tally(records.filter_map { |record| record[:status] }),
+        duration_ms: durations.sum.round(1),
+        p50_ms: percentile(durations, 0.50),
+        p95_ms: percentile(durations, 0.95),
+        max_ms: durations.max&.round(1),
+        requests_per_second: requests_per_second(records.count, records),
+        error_samples: errors.first(3).map { |record| record.slice(:status, :error, :body) }
+      }
+    end
+
+    def websocket_summary(client_count, records, baseline_connections, successful, observed_connections)
+      durations = records.filter_map { |record| record[:duration_ms] }
+      errors = records.reject { |record| record[:ok] }
+
+      {
+        clients: client_count,
+        opened: successful,
+        errors: errors.count,
+        connection_metric: observed_connections,
+        expected_connection_metric: successful,
+        baseline_connection_metric: baseline_connections,
+        p50_open_ms: percentile(durations, 0.50),
+        p95_open_ms: percentile(durations, 0.95),
+        max_open_ms: durations.max&.round(1),
+        error_samples: errors.first(3).map { |record| record.slice(:error) }
+      }
+    end
+
+    def failed_http_records(records)
+      records.select { |record| record[:error] || record[:status].to_i >= 400 }
+    end
+
+    def http_failures(level)
+      return [] unless level.fetch(:errors).positive?
+
+      ["HTTP #{level.fetch(:clients)} clients: #{level.fetch(:errors)} failed"]
+    end
+
+    def websocket_failures(level)
+      failures = []
+      if level.fetch(:errors).positive?
+        failures << "WS #{level.fetch(:clients)} clients: #{level.fetch(:errors)} failed"
+      end
+      if websocket_open_short?(level)
+        failures << "WS #{level.fetch(:clients)} clients: only #{level.fetch(:opened)} opened"
+      end
+      if websocket_metric_short?(level)
+        failures << "WS #{level.fetch(:clients)} clients: metrics saw #{level.fetch(:connection_metric)} connections"
+      end
+      failures
+    end
+
+    def websocket_open_short?(level)
+      level.fetch(:opened) < level.fetch(:clients)
+    end
+
+    def websocket_metric_short?(level)
+      level.fetch(:opened).positive? &&
+        level.fetch(:connection_metric).to_i < level.fetch(:expected_connection_metric).to_i
+    end
+
+    def summarize_tournament(tournament)
+      {
+        id: tournament.fetch(:id),
+        code: tournament.fetch(:code),
+        playoff_teams_amount: tournament.fetch(:playoff_teams_amount),
+        stage_count: tournament.fetch(:stages).count,
+        team_count: tournament.fetch(:teams).count
+      }
+    end
+
+    def error_body(response)
+      return nil if response.fetch(:status).to_i < 400
+
+      response.fetch(:json)
+    end
+
+    def requests_per_second(request_count, records)
+      level_duration_seconds = records.filter_map { |record| record[:level_duration_seconds] }.max.to_f
+      return nil unless level_duration_seconds.positive?
+
+      (request_count / level_duration_seconds).round(2)
+    end
+
+    def percentile(values, quantile)
+      sorted = values.compact.sort
+      return nil if sorted.empty?
+
+      index = [(sorted.length * quantile).ceil - 1, 0].max
+      sorted.fetch(index).round(1)
+    end
+
+    def tally(values)
+      values.each_with_object(Hash.new(0)) { |value, counts| counts[value.to_s] += 1 }
+    end
+
+    def drain_queue(queue)
+      items = []
+      items << queue.pop(true) until queue.empty?
+      items
+    rescue ThreadError
+      items
+    end
+
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def elapsed_ms(started_at)
+      ((monotonic_time - started_at) * 1000.0).round(1)
+    end
+  end
+end

e2e/lib/scenario_runner.rb

@@ -206,6 +206,20 @@ module TurniereE2E
       )
     end
 
+    def login_with_existing_credentials!
+      login_response = client.login!(email: @email, password: @password)
+      expect_status!(login_response, 200, "login #{@email}")
+      unless client.authenticated?
+        raise ApiError.new(
+          'login succeeded but no auth headers were returned',
+          status: 200,
+          body: login_response.fetch(:json)
+        )
+      end
+
+      { email: @email, username: @username || @email }
+    end
+
     def create_tournament!(payload)
       response = client.post('/tournaments', body: payload)
       expect_status!(response, 201, 'create tournament')
@@ -560,10 +574,3 @@ module TurniereE2E
     end
   end
 end
-    def login_with_existing_credentials!
-      login_response = client.login!(email: @email, password: @password)
-      expect_status!(login_response, 200, "login #{@email}")
-      raise ApiError.new('login succeeded but no auth headers were returned', status: 200, body: login_response.fetch(:json)) unless client.authenticated?
-
-      { email: @email, username: @username || @email }
-    end

e2e/load_test.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'json'
+require 'optparse'
+
+require_relative 'lib/load_test_runner'
+
+def parse_client_list(value)
+  return [] if value.nil? || value.strip == '' || value.strip.casecmp('none').zero?
+
+  value.split(',').map { |entry| Integer(entry.strip) }
+end
+
+options = {
+  base_url: ENV.fetch('TURNIERE_E2E_BASE_URL', 'http://127.0.0.1:3000'),
+  email: ENV['TURNIERE_E2E_EMAIL'],
+  password: ENV['TURNIERE_E2E_PASSWORD'],
+  username: ENV['TURNIERE_E2E_USERNAME'],
+  tournament_id: ENV['TURNIERE_LOAD_TOURNAMENT_ID'],
+  http_clients: parse_client_list(ENV.fetch('TURNIERE_LOAD_HTTP_CLIENTS', '1,10,50,100')),
+  websocket_clients: parse_client_list(ENV.fetch('TURNIERE_LOAD_WEBSOCKET_CLIENTS', '1,10,50,100,200')),
+  http_requests_per_client: Integer(ENV.fetch('TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT', '3')),
+  websocket_hold_seconds: Float(ENV.fetch('TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS', '1.0')),
+  websocket_connect_batch_size: Integer(ENV.fetch('TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE', '25')),
+  group_count: Integer(ENV.fetch('TURNIERE_LOAD_GROUP_COUNT', '4')),
+  teams_per_group: Integer(ENV.fetch('TURNIERE_LOAD_TEAMS_PER_GROUP', '4')),
+  playoff_teams_amount: Integer(ENV.fetch('TURNIERE_LOAD_PLAYOFF_TEAMS', '8')),
+  format: ENV.fetch('TURNIERE_LOAD_FORMAT', 'text')
+}
+
+# rubocop:disable Metrics/BlockLength
+OptionParser.new do |parser|
+  parser.banner = 'Usage: ruby e2e/load_test.rb [options]'
+
+  parser.on('--base-url URL', 'API base URL') { |value| options[:base_url] = value }
+  parser.on('--email EMAIL', 'Confirmed E2E user email') { |value| options[:email] = value }
+  parser.on('--password PASSWORD', 'Confirmed E2E user password') { |value| options[:password] = value }
+  parser.on('--username USERNAME', 'Confirmed E2E user username') { |value| options[:username] = value }
+  parser.on('--tournament-id ID', 'Existing tournament ID instead of creating one') do |value|
+    options[:tournament_id] = value
+  end
+  parser.on('--http-clients LIST', 'Comma-separated HTTP client levels, or none') do |value|
+    options[:http_clients] = parse_client_list(value)
+  end
+  parser.on('--websocket-clients LIST', 'Comma-separated websocket client levels, or none') do |value|
+    options[:websocket_clients] = parse_client_list(value)
+  end
+  parser.on('--http-requests-per-client COUNT', Integer, 'GET requests per HTTP client') do |value|
+    options[:http_requests_per_client] = value
+  end
+  parser.on('--websocket-hold-seconds SECONDS', Float, 'Seconds to hold websocket clients open') do |value|
+    options[:websocket_hold_seconds] = value
+  end
+  parser.on('--websocket-connect-batch-size COUNT', Integer, 'Concurrent websocket handshakes per batch') do |value|
+    options[:websocket_connect_batch_size] = value
+  end
+  parser.on('--group-count COUNT', Integer, 'Groups in generated tournament') { |value| options[:group_count] = value }
+  parser.on('--teams-per-group COUNT', Integer, 'Teams per generated group') do |value|
+    options[:teams_per_group] = value
+  end
+  parser.on('--playoff-teams-amount COUNT', Integer, 'Playoff teams in generated tournament') do |value|
+    options[:playoff_teams_amount] = value
+  end
+  parser.on('--format FORMAT', 'text or json') { |value| options[:format] = value }
+end.parse!
+# rubocop:enable Metrics/BlockLength
+
+runner_options = options.dup
+runner_options.delete(:format)
+runner = TurniereE2E::LoadTestRunner.new(**runner_options)
+result = runner.run
+
+if options[:format] == 'json'
+  puts JSON.pretty_generate(result)
+  exit
+end
+
+puts "Load test base_url=#{result.fetch(:base_url)} tournament_id=#{result.dig(:tournament, :id)}"
+
+result.fetch(:http).each do |level|
+  puts(
+    "HTTP #{level.fetch(:clients)} clients: " \
+    "#{level.fetch(:ok)}/#{level.fetch(:requests)} ok, errors=#{level.fetch(:errors)}, " \
+    "p50=#{level.fetch(:p50_ms)}ms, p95=#{level.fetch(:p95_ms)}ms, " \
+    "max=#{level.fetch(:max_ms)}ms, rps=#{level.fetch(:requests_per_second)}"
+  )
+end
+
+result.fetch(:websocket).each do |level|
+  puts(
+    "WS #{level.fetch(:clients)} clients: " \
+    "opened=#{level.fetch(:opened)}, errors=#{level.fetch(:errors)}, " \
+    "p50_open=#{level.fetch(:p50_open_ms)}ms, p95_open=#{level.fetch(:p95_open_ms)}ms, " \
+    "connections_metric=#{level.fetch(:connection_metric)}"
+  )
+end
+
+metrics = result.fetch(:metrics)
+puts(
+  "Metrics: action_cable_connections=#{metrics.fetch(:action_cable_connections)}, " \
+  "action_cable_connections_total=#{metrics.fetch(:action_cable_connections_total)}, " \
+  "ruby_threads=#{metrics.fetch(:ruby_threads)}"
+)

spec/e2e/http/load_test_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_relative '../../../e2e/lib/load_test_runner'
+
+RSpec.describe 'Load test HTTP E2E' do
+  required_env_vars = %w[TURNIERE_E2E_EMAIL TURNIERE_E2E_PASSWORD].freeze
+
+  before do
+    missing_vars = required_env_vars.reject { |name| ENV.key?(name) }
+    skip("HTTP E2E requires #{missing_vars.join(', ')}") unless missing_vars.empty?
+  end
+
+  let(:base_url) { ENV.fetch('TURNIERE_E2E_BASE_URL', 'http://127.0.0.1:3000') }
+
+  it 'exercises concurrent tournament reads and websocket subscriptions' do
+    result = TurniereE2E::LoadTestRunner.new(
+      base_url: base_url,
+      email: ENV.fetch('TURNIERE_E2E_EMAIL'),
+      password: ENV.fetch('TURNIERE_E2E_PASSWORD'),
+      username: ENV['TURNIERE_E2E_USERNAME'],
+      http_clients: [1, 2],
+      websocket_clients: [1, 2],
+      http_requests_per_client: 1,
+      websocket_hold_seconds: 0.1,
+      group_count: 1,
+      teams_per_group: 4,
+      playoff_teams_amount: 2
+    ).run
+
+    expect(result.fetch(:http).map { |level| level.fetch(:errors) }).to all(eq(0))
+    expect(result.fetch(:websocket).map { |level| level.fetch(:errors) }).to all(eq(0))
+    expect(result.fetch(:websocket).map { |level| level.fetch(:opened) }).to eq([1, 2])
+    expect(result.dig(:metrics, :action_cable_connections_total)).to be >= 2
+  end
+end

tasks.py

@@ -28,6 +28,14 @@ E2E_USERNAME = "e2e-user"
 E2E_ALT_EMAIL = "e2e-alt@example.com"
 E2E_ALT_PASSWORD = "password123"
 E2E_ALT_USERNAME = "e2e-alt-user"
+LOAD_HTTP_CLIENTS = "1,10,50,100"
+LOAD_WEBSOCKET_CLIENTS = "1,10,50,100,200"
+LOAD_HTTP_REQUESTS_PER_CLIENT = "3"
+LOAD_WEBSOCKET_HOLD_SECONDS = "1.0"
+LOAD_WEBSOCKET_CONNECT_BATCH_SIZE = "25"
+LOAD_GROUP_COUNT = "4"
+LOAD_TEAMS_PER_GROUP = "4"
+LOAD_PLAYOFF_TEAMS = "8"
 
 PRODUCTION_TAG = "registry.gitlab.com/turniere/turniere-backend/production/commits:local"
 TEST_TAG = "registry.gitlab.com/turniere/turniere-backend/test/commits:local"
@@ -526,6 +534,75 @@ def _run_blackbox_rspec(base_url, email, password, username, alt_email=E2E_ALT_E
         )
 
 
+def _load_env_value(value, env_name, default):
+    value = str(value or "").strip()
+    if value:
+        return value
+    return os.environ.get(env_name, default)
+
+
+def _run_load_test(
+    c,
+    base_url=TEST_BASE_URL,
+    email=E2E_EMAIL,
+    password=E2E_PASSWORD,
+    username=E2E_USERNAME,
+    tournament_id="",
+    http_clients="",
+    websocket_clients="",
+    http_requests_per_client="",
+    websocket_hold_seconds="",
+    websocket_connect_batch_size="",
+    group_count="",
+    teams_per_group="",
+    playoff_teams_amount="",
+    format="text",
+):
+    env = _env(
+        TURNIERE_E2E_BASE_URL=base_url,
+        TURNIERE_E2E_EMAIL=email,
+        TURNIERE_E2E_PASSWORD=password,
+        TURNIERE_E2E_USERNAME=username,
+        TURNIERE_LOAD_HTTP_CLIENTS=_load_env_value(http_clients, "TURNIERE_LOAD_HTTP_CLIENTS", LOAD_HTTP_CLIENTS),
+        TURNIERE_LOAD_WEBSOCKET_CLIENTS=_load_env_value(
+            websocket_clients,
+            "TURNIERE_LOAD_WEBSOCKET_CLIENTS",
+            LOAD_WEBSOCKET_CLIENTS,
+        ),
+        TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT=_load_env_value(
+            http_requests_per_client,
+            "TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT",
+            LOAD_HTTP_REQUESTS_PER_CLIENT,
+        ),
+        TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS=_load_env_value(
+            websocket_hold_seconds,
+            "TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS",
+            LOAD_WEBSOCKET_HOLD_SECONDS,
+        ),
+        TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE=_load_env_value(
+            websocket_connect_batch_size,
+            "TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE",
+            LOAD_WEBSOCKET_CONNECT_BATCH_SIZE,
+        ),
+        TURNIERE_LOAD_GROUP_COUNT=_load_env_value(group_count, "TURNIERE_LOAD_GROUP_COUNT", LOAD_GROUP_COUNT),
+        TURNIERE_LOAD_TEAMS_PER_GROUP=_load_env_value(
+            teams_per_group,
+            "TURNIERE_LOAD_TEAMS_PER_GROUP",
+            LOAD_TEAMS_PER_GROUP,
+        ),
+        TURNIERE_LOAD_PLAYOFF_TEAMS=_load_env_value(
+            playoff_teams_amount,
+            "TURNIERE_LOAD_PLAYOFF_TEAMS",
+            LOAD_PLAYOFF_TEAMS,
+        ),
+        TURNIERE_LOAD_FORMAT=format,
+    )
+    if tournament_id:
+        env["TURNIERE_LOAD_TOURNAMENT_ID"] = str(tournament_id)
+
+    c.run(_bundle_exec("ruby", "e2e/load_test.rb"), env=env, pty=True)
+
+
 def _start_server(command, env, log_path="/tmp/turniere-e2e-server.log"):
     log_handle = open(log_path, "w")
     process = subprocess.Popen(
@@ -808,6 +885,92 @@ def verify_http(
         _stop_process(process, log_handle)
 
 
+@task(name="load-test")
+def load_test(
+    c,
+    base_url=TEST_BASE_URL,
+    email=E2E_EMAIL,
+    password=E2E_PASSWORD,
+    username=E2E_USERNAME,
+    tournament_id="",
+    http_clients="",
+    websocket_clients="",
+    http_requests_per_client="",
+    websocket_hold_seconds="",
+    websocket_connect_batch_size="",
+    group_count="",
+    teams_per_group="",
+    playoff_teams_amount="",
+    format="text",
+):
+    """Run HTTP read and ActionCable websocket load smoke tests against a running backend."""
+    _run_load_test(
+        c,
+        base_url=base_url,
+        email=email,
+        password=password,
+        username=username,
+        tournament_id=tournament_id,
+        http_clients=http_clients,
+        websocket_clients=websocket_clients,
+        http_requests_per_client=http_requests_per_client,
+        websocket_hold_seconds=websocket_hold_seconds,
+        websocket_connect_batch_size=websocket_connect_batch_size,
+        group_count=group_count,
+        teams_per_group=teams_per_group,
+        playoff_teams_amount=playoff_teams_amount,
+        format=format,
+    )
+
+
+@task(name="verify-load-test")
+def verify_load_test(
+    c,
+    base_url=TEST_BASE_URL,
+    email=E2E_EMAIL,
+    password=E2E_PASSWORD,
+    username=E2E_USERNAME,
+    reset_db=True,
+    http_clients="",
+    websocket_clients="",
+    http_requests_per_client="",
+    websocket_hold_seconds="",
+    websocket_connect_batch_size="",
+    group_count="",
+    teams_per_group="",
+    playoff_teams_amount="",
+):
+    """Boot a local test server, then run HTTP/websocket load smoke tests against it."""
+    if reset_db:
+        db_reset(c, env="test")
+    db_migrate(c, env="test")
+    bootstrap_e2e_user(c, email=email, password=password, username=username)
+    parsed_base_url = urlparse(base_url)
+    server_host = parsed_base_url.hostname or "127.0.0.1"
+    server_port = parsed_base_url.port or 3000
+
+    process, log_handle = _start_test_server(host=server_host, port=server_port)
+    try:
+        _wait_for_http(base_url=base_url, timeout=60)
+        load_test(
+            c,
+            base_url=base_url,
+            email=email,
+            password=password,
+            username=username,
+            http_clients=http_clients,
+            websocket_clients=websocket_clients,
+            http_requests_per_client=http_requests_per_client,
+            websocket_hold_seconds=websocket_hold_seconds,
+            websocket_connect_batch_size=websocket_connect_batch_size,
+            group_count=group_count,
+            teams_per_group=teams_per_group,
+            playoff_teams_amount=playoff_teams_amount,
+        )
+    finally:
+        _stop_process(process, log_handle)
+
+
 @task
 def verify(c):
     """Run lint, normal specs, and HTTP E2E locally."""
@@ -1744,6 +1907,59 @@ def blackbox_service(
     _run_blackbox_rspec(base_url, email, password, username, alt_email, alt_password, alt_username)
 
 
+@task(name="blackbox-load-service")
+def blackbox_load_service(
+    c,
+    base_url=BLACKBOX_INTERNAL_BASE_URL,
+    email=E2E_EMAIL,
+    password=E2E_PASSWORD,
+    username=E2E_USERNAME,
+    http_clients="",
+    websocket_clients="",
+    http_requests_per_client="",
+    websocket_hold_seconds="",
+    websocket_connect_batch_size="",
+    group_count="",
+    teams_per_group="",
+    playoff_teams_amount="",
+):
+    """Run load smoke tests against an already running production app plus Postgres sidecar."""
+    _print_header("Running production database migrations")
+    c.run("bundle exec rails db:migrate", env=_shared_production_env(), pty=True)
+
+    _print_header(f"Waiting for app healthcheck at {base_url}")
+    _wait_for_http(base_url=base_url, timeout=120)
+
+    _print_header("Bootstrapping confirmed E2E user")
+    _bootstrap_blackbox_users(
+        c,
+        _shared_production_env(),
+        email,
+        password,
+        username,
+        E2E_ALT_EMAIL,
+        E2E_ALT_PASSWORD,
+        E2E_ALT_USERNAME,
+    )
+
+    _print_header("Running HTTP/websocket load smoke against production image")
+    _run_load_test(
+        c,
+        base_url=base_url,
+        email=email,
+        password=password,
+        username=username,
+        http_clients=http_clients,
+        websocket_clients=websocket_clients,
+        http_requests_per_client=http_requests_per_client,
+        websocket_hold_seconds=websocket_hold_seconds,
+        websocket_connect_batch_size=websocket_connect_batch_size,
+        group_count=group_count,
+        teams_per_group=teams_per_group,
+        playoff_teams_amount=playoff_teams_amount,
+    )
+
+
 @task(name="blackbox-follow-service")
 def blackbox_follow_service(
     c,