From 3627c1a3ce1ef88e316e264058f6931220e6c2be Mon Sep 17 00:00:00 2001 From: Malaber Date: Wed, 29 Apr 2026 22:34:32 +0200 Subject: [PATCH] feat(load): add load smoke tests --- .gitlab-ci.yml | 60 ++++++ doc/load_tests.md | 39 ++++ e2e/lib/load_test_runner.rb | 342 ++++++++++++++++++++++++++++++++ e2e/lib/scenario_runner.rb | 21 +- e2e/load_test.rb | 104 ++++++++++ spec/e2e/http/load_test_spec.rb | 36 ++++ tasks.py | 216 ++++++++++++++++++++ 7 files changed, 811 insertions(+), 7 deletions(-) create mode 100644 doc/load_tests.md create mode 100644 e2e/lib/load_test_runner.rb create mode 100755 e2e/load_test.rb create mode 100644 spec/e2e/http/load_test_spec.rb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a52e46..99034f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,6 +98,66 @@ rails e2e spec: - cd /app - inv blackbox-service +rails load smoke: + stage: e2e + tags: + - docker + needs: + - job: build_image + image: $CI_REGISTRY_IMAGE/test/commits:$CI_COMMIT_SHA + services: + - name: postgres:16 + alias: postgres + - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA + alias: app + variables: + TURNIERE_DISABLE_EMAIL_DELIVERY: "1" + command: + - bundle + - exec + - rails + - s + - -p + - "3000" + - -b + - 0.0.0.0 + variables: + FF_NETWORK_PER_BUILD: "true" + POSTGRES_DB: turniere_blackbox + POSTGRES_USER: turniere + POSTGRES_PASSWORD: turniere + POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox + POSTGRES_USERNAME: turniere + SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod + MAILGUN_API_KEY: blackbox-test-api-key + MAILGUN_DOMAIN: blackbox.example.com + RAILS_LOG_TO_STDOUT: "1" + RAILS_SERVE_STATIC_FILES: "1" + NO_PROXY: app,postgres,localhost,127.0.0.1 + no_proxy: app,postgres,localhost,127.0.0.1 + TURNIERE_E2E_BASE_URL: http://app:3000 + TURNIERE_E2E_EMAIL: e2e@example.com + TURNIERE_E2E_PASSWORD: password123 + TURNIERE_E2E_USERNAME: e2e-user + TURNIERE_LOAD_HTTP_CLIENTS: "1,10,50,100" + TURNIERE_LOAD_WEBSOCKET_CLIENTS: "1,10,50,100,200" + TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT: "3" + TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS: "1.0" + TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE: "25" + rules: + - if: $SKIP_TEST + when: never + - if: $SKIP_ALL_SPECS + when: never + - if: $SKIP_LOAD_SPEC + when: never + - if: $CI_PIPELINE_SOURCE != "push" + when: never + - when: always + script: + - cd /app + - inv blackbox-load-service + rails follow sync e2e spec: stage: e2e tags: diff --git a/doc/load_tests.md b/doc/load_tests.md new file mode 100644 index 0000000..eec9039 --- /dev/null +++ b/doc/load_tests.md @@ -0,0 +1,39 @@ +# Load Tests + +Load smoke tests live in `e2e/load_test.rb` and run through invoke tasks. + +The runner creates a realistic group-stage tournament over HTTP, then measures: + +- concurrent `GET /tournaments/:id` reads at each configured client level +- active ActionCable subscriptions to `TournamentChannel`, opened in bounded + batches so the test measures 200 active sockets instead of only handshake + burst capacity +- `/metrics` websocket gauges and process counters after the run + +It prints relative timings such as p50, p95, max, and requests per second. The +CI goal is not a fixed benchmark. The job fails when requests error, websocket +clients cannot subscribe, or ActionCable connection metrics do not reflect the +opened clients. + +Useful commands: + +- `inv verify-load-test` +- `inv load-test --base-url=http://127.0.0.1:3000` +- `inv load-test --http-clients=1,10,50,100 --websocket-clients=1,10,50,100,200` +- `inv blackbox-load-service` + +Useful environment overrides: + +- `TURNIERE_LOAD_HTTP_CLIENTS=1,10,50,100` +- `TURNIERE_LOAD_WEBSOCKET_CLIENTS=1,10,50,100,200` +- `TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT=3` +- `TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS=1.0` +- `TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE=25` +- `TURNIERE_LOAD_GROUP_COUNT=4` +- `TURNIERE_LOAD_TEAMS_PER_GROUP=4` +- `TURNIERE_LOAD_PLAYOFF_TEAMS=8` +- `TURNIERE_LOAD_FORMAT=json` + +GitLab CI runs `rails load smoke` against the production image with Postgres, +covering read pressure up to 100 HTTP clients and active websocket pressure up +to 200 clients. diff --git a/e2e/lib/load_test_runner.rb b/e2e/lib/load_test_runner.rb new file mode 100644 index 0000000..ddcb77f --- /dev/null +++ b/e2e/lib/load_test_runner.rb @@ -0,0 +1,342 @@ +# frozen_string_literal: true + +require 'json' +require 'time' + +require_relative 'action_cable_client' +require_relative 'api_client' +require_relative 'scenario_runner' + +module TurniereE2E + class LoadTestRunner + DEFAULT_HTTP_CLIENTS = [1, 10, 50, 100].freeze + DEFAULT_WEBSOCKET_CLIENTS = [1, 10, 50, 100, 200].freeze + + def initialize(**options) + @base_url = options.fetch(:base_url).delete_suffix('/') + @email = options[:email] + @password = options[:password] + @username = options[:username] + @tournament_id = options[:tournament_id] + @http_clients = options.fetch(:http_clients, DEFAULT_HTTP_CLIENTS) + @websocket_clients = options.fetch(:websocket_clients, DEFAULT_WEBSOCKET_CLIENTS) + @http_requests_per_client = options.fetch(:http_requests_per_client, 3).to_i + @websocket_hold_seconds = options.fetch(:websocket_hold_seconds, 1.0).to_f + @websocket_connect_batch_size = options.fetch(:websocket_connect_batch_size, 25).to_i + @group_count = options.fetch(:group_count, 4).to_i + @teams_per_group = options.fetch(:teams_per_group, 4).to_i + @playoff_teams_amount = options.fetch(:playoff_teams_amount, 8).to_i + @fail_on_error = options.fetch(:fail_on_error, true) + end + + def run + tournament = prepare_tournament + result = { + base_url: base_url, + tournament: tournament, + http: http_clients.map { |client_count| run_http_level(tournament, client_count) }, + websocket: websocket_clients.map { |client_count| run_websocket_level(tournament, client_count) }, + metrics: metrics_summary + } + + assert_success!(result) if fail_on_error + result + end + + private + + attr_reader :base_url, :email, :password, :username, :tournament_id, :http_clients, :websocket_clients, + :http_requests_per_client, :websocket_hold_seconds, :websocket_connect_batch_size, :group_count, + :teams_per_group, :playoff_teams_amount, :fail_on_error + + def prepare_tournament + return fetch_existing_tournament if tournament_id.to_s.strip != '' + + runner = ScenarioRunner.new(base_url: base_url, email: email, password: password, username: username) + runner.wait_for_healthcheck! + result = runner.run_group_stage_render_profile( + group_count: group_count, + teams_per_group: teams_per_group, + playoff_teams_amount: playoff_teams_amount, + stop_at: :created + ) + result.fetch(:tournament) + end + + def fetch_existing_tournament + client = ApiClient.new(base_url: base_url) + response = client.get("/tournaments/#{tournament_id}") + unless response.fetch(:status) == 200 + raise ApiError.new( + "load-test tournament fetch failed with #{response.fetch(:status)}", + status: response.fetch(:status), + body: response.fetch(:json) + ) + end + + summarize_tournament(response.fetch(:json)) + end + + def run_http_level(tournament, client_count) + path = "/tournaments/#{tournament.fetch(:id)}" + records = run_threads(client_count) do + client = ApiClient.new(base_url: base_url) + Array.new(http_requests_per_client) { measure_http_request(client, path) } + end.flatten + + http_summary(client_count, records) + end + + def run_websocket_level(tournament, client_count) + clients = [] + baseline_connections = metric_scalar('turniere_action_cable_connections').to_i + records = run_websocket_open_batches(tournament.fetch(:id), client_count, clients) + successful = records.count { |record| record[:ok] } + observed_connections = wait_for_connection_metric(successful) + sleep websocket_hold_seconds if websocket_hold_seconds.positive? + + websocket_summary(client_count, records, baseline_connections, successful, observed_connections) + ensure + clients.each(&:close) + end + + def run_websocket_open_batches(tournament_id, client_count, clients) + batch_size = websocket_connect_batch_size.positive? ? websocket_connect_batch_size : client_count + + Array.new(client_count) { true }.each_slice(batch_size).flat_map do |batch| + run_threads(batch.count) do + measure_websocket_open(tournament_id).tap do |record| + clients << record.delete(:client) if record[:client] + end + end + end + end + + def run_threads(count) + return [] if count.to_i <= 0 + + ready = Queue.new + start = Queue.new + results = Queue.new + threads = Array.new(count) do + Thread.new do + ready << true + start.pop + results << yield + rescue StandardError => e + results << [{ error: "#{e.class}: #{e.message}" }] + end + end + + count.times { ready.pop } + started_at = monotonic_time + count.times { start << true } + threads.each(&:join) + finished_at = monotonic_time + + drain_queue(results).flatten.map do |record| + record.is_a?(Hash) ? record.merge(level_duration_seconds: finished_at - started_at) : record + end + end + + def measure_http_request(client, path) + started_at = monotonic_time + response = client.get(path) + { + status: response.fetch(:status), + duration_ms: elapsed_ms(started_at), + body: error_body(response) + }.compact + rescue StandardError => e + { error: "#{e.class}: #{e.message}", duration_ms: elapsed_ms(started_at) } + end + + def measure_websocket_open(tournament_id) + client = ActionCableClient.new(base_url: base_url) + started_at = monotonic_time + client.subscribe_tournament!(tournament_id: tournament_id) + payload = client.wait_for_tournament_payload!(timeout: 20) + { + ok: true, + duration_ms: elapsed_ms(started_at), + payload_type: payload.fetch('type'), + client: client + } + rescue StandardError => e + client&.close + { ok: false, duration_ms: elapsed_ms(started_at), error: "#{e.class}: #{e.message}" } + end + + def wait_for_connection_metric(expected, timeout: 10) + deadline = monotonic_time + timeout + loop do + value = metric_scalar('turniere_action_cable_connections') + return value if value && value >= expected + return value if monotonic_time >= deadline + + sleep 0.2 + end + end + + def metrics_summary + { + action_cable_connections: metric_scalar('turniere_action_cable_connections'), + action_cable_connections_total: metric_scalar('turniere_action_cable_connections_total'), + ruby_threads: metric_scalar('turniere_ruby_threads'), + active_record_connections: metric_samples('turniere_active_record_connection_pool') + } + end + + def metric_scalar(name) + metric_samples(name).map { |sample| sample.fetch(:value) }.max + end + + def metric_samples(name) + body = metrics_body + body.lines.filter_map do |line| + next if line.start_with?('#') + + sample, value = line.split(/\s+/, 2) + next unless sample == name || sample.start_with?("#{name}{") + + { sample: sample, value: value.to_f } + end + end + + def metrics_body + response = ApiClient.new(base_url: base_url).get('/metrics') + return '' unless response.fetch(:status) == 200 + + response.dig(:json, :raw_body).to_s + end + + def assert_success!(result) + failures = [] + result.fetch(:http).each { |level| failures.concat(http_failures(level)) } + result.fetch(:websocket).each { |level| failures.concat(websocket_failures(level)) } + + raise "load test failed: #{failures.join('; ')}" unless failures.empty? + end + + def http_summary(client_count, records) + durations = records.filter_map { |record| record[:duration_ms] } + errors = failed_http_records(records) + + { + clients: client_count, + requests: records.count, + ok: records.count - errors.count, + errors: errors.count, + statuses: tally(records.filter_map { |record| record[:status] }), + duration_ms: durations.sum.round(1), + p50_ms: percentile(durations, 0.50), + p95_ms: percentile(durations, 0.95), + max_ms: durations.max&.round(1), + requests_per_second: requests_per_second(records.count, records), + error_samples: errors.first(3).map { |record| record.slice(:status, :error, :body) } + } + end + + def websocket_summary(client_count, records, baseline_connections, successful, observed_connections) + durations = records.filter_map { |record| record[:duration_ms] } + errors = records.reject { |record| record[:ok] } + + { + clients: client_count, + opened: successful, + errors: errors.count, + connection_metric: observed_connections, + expected_connection_metric: successful, + baseline_connection_metric: baseline_connections, + p50_open_ms: percentile(durations, 0.50), + p95_open_ms: percentile(durations, 0.95), + max_open_ms: durations.max&.round(1), + error_samples: errors.first(3).map { |record| record.slice(:error) } + } + end + + def failed_http_records(records) + records.select { |record| record[:error] || record[:status].to_i >= 400 } + end + + def http_failures(level) + return [] unless level.fetch(:errors).positive? + + ["HTTP #{level.fetch(:clients)} clients: #{level.fetch(:errors)} failed"] + end + + def websocket_failures(level) + failures = [] + if level.fetch(:errors).positive? + failures << "WS #{level.fetch(:clients)} clients: #{level.fetch(:errors)} failed" + end + if websocket_open_short?(level) + failures << "WS #{level.fetch(:clients)} clients: only #{level.fetch(:opened)} opened" + end + if websocket_metric_short?(level) + failures << "WS #{level.fetch(:clients)} clients: metrics saw #{level.fetch(:connection_metric)} connections" + end + failures + end + + def websocket_open_short?(level) + level.fetch(:opened) < level.fetch(:clients) + end + + def websocket_metric_short?(level) + level.fetch(:opened).positive? && + level.fetch(:connection_metric).to_i < level.fetch(:expected_connection_metric).to_i + end + + def summarize_tournament(tournament) + { + id: tournament.fetch(:id), + code: tournament.fetch(:code), + playoff_teams_amount: tournament.fetch(:playoff_teams_amount), + stage_count: tournament.fetch(:stages).count, + team_count: tournament.fetch(:teams).count + } + end + + def error_body(response) + return nil if response.fetch(:status).to_i < 400 + + response.fetch(:json) + end + + def requests_per_second(request_count, records) + level_duration_seconds = records.filter_map { |record| record[:level_duration_seconds] }.max.to_f + return nil unless level_duration_seconds.positive? + + (request_count / level_duration_seconds).round(2) + end + + def percentile(values, quantile) + sorted = values.compact.sort + return nil if sorted.empty? + + index = [(sorted.length * quantile).ceil - 1, 0].max + sorted.fetch(index).round(1) + end + + def tally(values) + values.each_with_object(Hash.new(0)) { |value, counts| counts[value.to_s] += 1 } + end + + def drain_queue(queue) + items = [] + items << queue.pop(true) until queue.empty? + items + rescue ThreadError + items + end + + def monotonic_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def elapsed_ms(started_at) + ((monotonic_time - started_at) * 1000.0).round(1) + end + end +end diff --git a/e2e/lib/scenario_runner.rb b/e2e/lib/scenario_runner.rb index 664db82..f3d3eb5 100644 --- a/e2e/lib/scenario_runner.rb +++ b/e2e/lib/scenario_runner.rb @@ -206,6 +206,20 @@ module TurniereE2E ) end + def login_with_existing_credentials! + login_response = client.login!(email: @email, password: @password) + expect_status!(login_response, 200, "login #{@email}") + unless client.authenticated? + raise ApiError.new( + 'login succeeded but no auth headers were returned', + status: 200, + body: login_response.fetch(:json) + ) + end + + { email: @email, username: @username || @email } + end + def create_tournament!(payload) response = client.post('/tournaments', body: payload) expect_status!(response, 201, 'create tournament') @@ -560,10 +574,3 @@ module TurniereE2E end end end - def login_with_existing_credentials! - login_response = client.login!(email: @email, password: @password) - expect_status!(login_response, 200, "login #{@email}") - raise ApiError.new('login succeeded but no auth headers were returned', status: 200, body: login_response.fetch(:json)) unless client.authenticated? - - { email: @email, username: @username || @email } - end diff --git a/e2e/load_test.rb b/e2e/load_test.rb new file mode 100755 index 0000000..1ca9e07 --- /dev/null +++ b/e2e/load_test.rb @@ -0,0 +1,104 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' +require 'optparse' + +require_relative 'lib/load_test_runner' + +def parse_client_list(value) + return [] if value.nil? || value.strip == '' || value.strip.casecmp('none').zero? + + value.split(',').map { |entry| Integer(entry.strip) } +end + +options = { + base_url: ENV.fetch('TURNIERE_E2E_BASE_URL', 'http://127.0.0.1:3000'), + email: ENV['TURNIERE_E2E_EMAIL'], + password: ENV['TURNIERE_E2E_PASSWORD'], + username: ENV['TURNIERE_E2E_USERNAME'], + tournament_id: ENV['TURNIERE_LOAD_TOURNAMENT_ID'], + http_clients: parse_client_list(ENV.fetch('TURNIERE_LOAD_HTTP_CLIENTS', '1,10,50,100')), + websocket_clients: parse_client_list(ENV.fetch('TURNIERE_LOAD_WEBSOCKET_CLIENTS', '1,10,50,100,200')), + http_requests_per_client: Integer(ENV.fetch('TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT', '3')), + websocket_hold_seconds: Float(ENV.fetch('TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS', '1.0')), + websocket_connect_batch_size: Integer(ENV.fetch('TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE', '25')), + group_count: Integer(ENV.fetch('TURNIERE_LOAD_GROUP_COUNT', '4')), + teams_per_group: Integer(ENV.fetch('TURNIERE_LOAD_TEAMS_PER_GROUP', '4')), + playoff_teams_amount: Integer(ENV.fetch('TURNIERE_LOAD_PLAYOFF_TEAMS', '8')), + format: ENV.fetch('TURNIERE_LOAD_FORMAT', 'text') +} + +# rubocop:disable Metrics/BlockLength +OptionParser.new do |parser| + parser.banner = 'Usage: ruby e2e/load_test.rb [options]' + + parser.on('--base-url URL', 'API base URL') { |value| options[:base_url] = value } + parser.on('--email EMAIL', 'Confirmed E2E user email') { |value| options[:email] = value } + parser.on('--password PASSWORD', 'Confirmed E2E user password') { |value| options[:password] = value } + parser.on('--username USERNAME', 'Confirmed E2E user username') { |value| options[:username] = value } + parser.on('--tournament-id ID', 'Existing tournament ID instead of creating one') do |value| + options[:tournament_id] = value + end + parser.on('--http-clients LIST', 'Comma-separated HTTP client levels, or none') do |value| + options[:http_clients] = parse_client_list(value) + end + parser.on('--websocket-clients LIST', 'Comma-separated websocket client levels, or none') do |value| + options[:websocket_clients] = parse_client_list(value) + end + parser.on('--http-requests-per-client COUNT', Integer, 'GET requests per HTTP client') do |value| + options[:http_requests_per_client] = value + end + parser.on('--websocket-hold-seconds SECONDS', Float, 'Seconds to hold websocket clients open') do |value| + options[:websocket_hold_seconds] = value + end + parser.on('--websocket-connect-batch-size COUNT', Integer, 'Concurrent websocket handshakes per batch') do |value| + options[:websocket_connect_batch_size] = value + end + parser.on('--group-count COUNT', Integer, 'Groups in generated tournament') { |value| options[:group_count] = value } + parser.on('--teams-per-group COUNT', Integer, 'Teams per generated group') do |value| + options[:teams_per_group] = value + end + parser.on('--playoff-teams-amount COUNT', Integer, 'Playoff teams in generated tournament') do |value| + options[:playoff_teams_amount] = value + end + parser.on('--format FORMAT', 'text or json') { |value| options[:format] = value } +end.parse! +# rubocop:enable Metrics/BlockLength + +runner_options = options.dup +runner_options.delete(:format) +runner = TurniereE2E::LoadTestRunner.new(**runner_options) +result = runner.run + +if options[:format] == 'json' + puts JSON.pretty_generate(result) + exit +end + +puts "Load test base_url=#{result.fetch(:base_url)} tournament_id=#{result.dig(:tournament, :id)}" + +result.fetch(:http).each do |level| + puts( + "HTTP #{level.fetch(:clients)} clients: " \ + "#{level.fetch(:ok)}/#{level.fetch(:requests)} ok, errors=#{level.fetch(:errors)}, " \ + "p50=#{level.fetch(:p50_ms)}ms, p95=#{level.fetch(:p95_ms)}ms, " \ + "max=#{level.fetch(:max_ms)}ms, rps=#{level.fetch(:requests_per_second)}" + ) +end + +result.fetch(:websocket).each do |level| + puts( + "WS #{level.fetch(:clients)} clients: " \ + "opened=#{level.fetch(:opened)}, errors=#{level.fetch(:errors)}, " \ + "p50_open=#{level.fetch(:p50_open_ms)}ms, p95_open=#{level.fetch(:p95_open_ms)}ms, " \ + "connections_metric=#{level.fetch(:connection_metric)}" + ) +end + +metrics = result.fetch(:metrics) +puts( + "Metrics: action_cable_connections=#{metrics.fetch(:action_cable_connections)}, " \ + "action_cable_connections_total=#{metrics.fetch(:action_cable_connections_total)}, " \ + "ruby_threads=#{metrics.fetch(:ruby_threads)}" +) diff --git a/spec/e2e/http/load_test_spec.rb b/spec/e2e/http/load_test_spec.rb new file mode 100644 index 0000000..c7299af --- /dev/null +++ b/spec/e2e/http/load_test_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../../../e2e/lib/load_test_runner' + +RSpec.describe 'Load test HTTP E2E' do + required_env_vars = %w[TURNIERE_E2E_EMAIL TURNIERE_E2E_PASSWORD].freeze + + before do + missing_vars = required_env_vars.reject { |name| ENV.key?(name) } + skip("HTTP E2E requires #{missing_vars.join(', ')}") unless missing_vars.empty? + end + + let(:base_url) { ENV.fetch('TURNIERE_E2E_BASE_URL', 'http://127.0.0.1:3000') } + + it 'exercises concurrent tournament reads and websocket subscriptions' do + result = TurniereE2E::LoadTestRunner.new( + base_url: base_url, + email: ENV.fetch('TURNIERE_E2E_EMAIL'), + password: ENV.fetch('TURNIERE_E2E_PASSWORD'), + username: ENV['TURNIERE_E2E_USERNAME'], + http_clients: [1, 2], + websocket_clients: [1, 2], + http_requests_per_client: 1, + websocket_hold_seconds: 0.1, + group_count: 1, + teams_per_group: 4, + playoff_teams_amount: 2 + ).run + + expect(result.fetch(:http).map { |level| level.fetch(:errors) }).to all(eq(0)) + expect(result.fetch(:websocket).map { |level| level.fetch(:errors) }).to all(eq(0)) + expect(result.fetch(:websocket).map { |level| level.fetch(:opened) }).to eq([1, 2]) + expect(result.dig(:metrics, :action_cable_connections_total)).to be >= 2 + end +end diff --git a/tasks.py b/tasks.py index 299b6a3..c1f323a 100644 --- a/tasks.py +++ b/tasks.py @@ -28,6 +28,14 @@ E2E_USERNAME = "e2e-user" E2E_ALT_EMAIL = "e2e-alt@example.com" E2E_ALT_PASSWORD = "password123" E2E_ALT_USERNAME = "e2e-alt-user" +LOAD_HTTP_CLIENTS = "1,10,50,100" +LOAD_WEBSOCKET_CLIENTS = "1,10,50,100,200" +LOAD_HTTP_REQUESTS_PER_CLIENT = "3" +LOAD_WEBSOCKET_HOLD_SECONDS = "1.0" +LOAD_WEBSOCKET_CONNECT_BATCH_SIZE = "25" +LOAD_GROUP_COUNT = "4" +LOAD_TEAMS_PER_GROUP = "4" +LOAD_PLAYOFF_TEAMS = "8" PRODUCTION_TAG = "registry.gitlab.com/turniere/turniere-backend/production/commits:local" TEST_TAG = "registry.gitlab.com/turniere/turniere-backend/test/commits:local" @@ -526,6 +534,75 @@ def _run_blackbox_rspec(base_url, email, password, username, alt_email=E2E_ALT_E ) +def _load_env_value(value, env_name, default): + value = str(value or "").strip() + if value: + return value + return os.environ.get(env_name, default) + + +def _run_load_test( + c, + base_url=TEST_BASE_URL, + email=E2E_EMAIL, + password=E2E_PASSWORD, + username=E2E_USERNAME, + tournament_id="", + http_clients="", + websocket_clients="", + http_requests_per_client="", + websocket_hold_seconds="", + websocket_connect_batch_size="", + group_count="", + teams_per_group="", + playoff_teams_amount="", + format="text", +): + env = _env( + TURNIERE_E2E_BASE_URL=base_url, + TURNIERE_E2E_EMAIL=email, + TURNIERE_E2E_PASSWORD=password, + TURNIERE_E2E_USERNAME=username, + TURNIERE_LOAD_HTTP_CLIENTS=_load_env_value(http_clients, "TURNIERE_LOAD_HTTP_CLIENTS", LOAD_HTTP_CLIENTS), + TURNIERE_LOAD_WEBSOCKET_CLIENTS=_load_env_value( + websocket_clients, + "TURNIERE_LOAD_WEBSOCKET_CLIENTS", + LOAD_WEBSOCKET_CLIENTS, + ), + TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT=_load_env_value( + http_requests_per_client, + "TURNIERE_LOAD_HTTP_REQUESTS_PER_CLIENT", + LOAD_HTTP_REQUESTS_PER_CLIENT, + ), + TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS=_load_env_value( + websocket_hold_seconds, + "TURNIERE_LOAD_WEBSOCKET_HOLD_SECONDS", + LOAD_WEBSOCKET_HOLD_SECONDS, + ), + TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE=_load_env_value( + websocket_connect_batch_size, + "TURNIERE_LOAD_WEBSOCKET_CONNECT_BATCH_SIZE", + LOAD_WEBSOCKET_CONNECT_BATCH_SIZE, + ), + TURNIERE_LOAD_GROUP_COUNT=_load_env_value(group_count, "TURNIERE_LOAD_GROUP_COUNT", LOAD_GROUP_COUNT), + TURNIERE_LOAD_TEAMS_PER_GROUP=_load_env_value( + teams_per_group, + "TURNIERE_LOAD_TEAMS_PER_GROUP", + LOAD_TEAMS_PER_GROUP, + ), + TURNIERE_LOAD_PLAYOFF_TEAMS=_load_env_value( + playoff_teams_amount, + "TURNIERE_LOAD_PLAYOFF_TEAMS", + LOAD_PLAYOFF_TEAMS, + ), + TURNIERE_LOAD_FORMAT=format, + ) + if tournament_id: + env["TURNIERE_LOAD_TOURNAMENT_ID"] = str(tournament_id) + + c.run(_bundle_exec("ruby", "e2e/load_test.rb"), env=env, pty=True) + + def _start_server(command, env, log_path="/tmp/turniere-e2e-server.log"): log_handle = open(log_path, "w") process = subprocess.Popen( @@ -808,6 +885,92 @@ def verify_http( _stop_process(process, log_handle) +@task(name="load-test") +def load_test( + c, + base_url=TEST_BASE_URL, + email=E2E_EMAIL, + password=E2E_PASSWORD, + username=E2E_USERNAME, + tournament_id="", + http_clients="", + websocket_clients="", + http_requests_per_client="", + websocket_hold_seconds="", + websocket_connect_batch_size="", + group_count="", + teams_per_group="", + playoff_teams_amount="", + format="text", +): + """Run HTTP read and ActionCable websocket load smoke tests against a running backend.""" + _run_load_test( + c, + base_url=base_url, + email=email, + password=password, + username=username, + tournament_id=tournament_id, + http_clients=http_clients, + websocket_clients=websocket_clients, + http_requests_per_client=http_requests_per_client, + websocket_hold_seconds=websocket_hold_seconds, + websocket_connect_batch_size=websocket_connect_batch_size, + group_count=group_count, + teams_per_group=teams_per_group, + playoff_teams_amount=playoff_teams_amount, + format=format, + ) + + +@task(name="verify-load-test") +def verify_load_test( + c, + base_url=TEST_BASE_URL, + email=E2E_EMAIL, + password=E2E_PASSWORD, + username=E2E_USERNAME, + reset_db=True, + http_clients="", + websocket_clients="", + http_requests_per_client="", + websocket_hold_seconds="", + websocket_connect_batch_size="", + group_count="", + teams_per_group="", + playoff_teams_amount="", +): + """Boot a local test server, then run HTTP/websocket load smoke tests against it.""" + if reset_db: + db_reset(c, env="test") + db_migrate(c, env="test") + bootstrap_e2e_user(c, email=email, password=password, username=username) + parsed_base_url = urlparse(base_url) + server_host = parsed_base_url.hostname or "127.0.0.1" + server_port = parsed_base_url.port or 3000 + + process, log_handle = _start_test_server(host=server_host, port=server_port) + try: + _wait_for_http(base_url=base_url, timeout=60) + load_test( + c, + base_url=base_url, + email=email, + password=password, + username=username, + http_clients=http_clients, + websocket_clients=websocket_clients, + http_requests_per_client=http_requests_per_client, + websocket_hold_seconds=websocket_hold_seconds, + websocket_connect_batch_size=websocket_connect_batch_size, + group_count=group_count, + teams_per_group=teams_per_group, + playoff_teams_amount=playoff_teams_amount, + ) + finally: + _stop_process(process, log_handle) + + @task def verify(c): """Run lint, normal specs, and HTTP E2E locally.""" @@ -1744,6 +1907,59 @@ def blackbox_service( _run_blackbox_rspec(base_url, email, password, username, alt_email, alt_password, alt_username) +@task(name="blackbox-load-service") +def blackbox_load_service( + c, + base_url=BLACKBOX_INTERNAL_BASE_URL, + email=E2E_EMAIL, + password=E2E_PASSWORD, + username=E2E_USERNAME, + http_clients="", + websocket_clients="", + http_requests_per_client="", + websocket_hold_seconds="", + websocket_connect_batch_size="", + group_count="", + teams_per_group="", + playoff_teams_amount="", +): + """Run load smoke tests against an already running production app plus Postgres sidecar.""" + _print_header("Running production database migrations") + c.run("bundle exec rails db:migrate", env=_shared_production_env(), pty=True) + + _print_header(f"Waiting for app healthcheck at {base_url}") + _wait_for_http(base_url=base_url, timeout=120) + + _print_header("Bootstrapping confirmed E2E user") + _bootstrap_blackbox_users( + c, + _shared_production_env(), + email, + password, + username, + E2E_ALT_EMAIL, + E2E_ALT_PASSWORD, + E2E_ALT_USERNAME, + ) + + _print_header("Running HTTP/websocket load smoke against production image") + _run_load_test( + c, + base_url=base_url, + email=email, + password=password, + username=username, + http_clients=http_clients, + websocket_clients=websocket_clients, + http_requests_per_client=http_requests_per_client, + websocket_hold_seconds=websocket_hold_seconds, + websocket_connect_batch_size=websocket_connect_batch_size, + group_count=group_count, + teams_per_group=teams_per_group, + playoff_teams_amount=playoff_teams_amount, + ) + + @task(name="blackbox-follow-service") def blackbox_follow_service( c,