diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e825c8..feaf8dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,30 +35,10 @@ rails spec: rails e2e spec: stage: test - image: $CI_REGISTRY_IMAGE/test/commits:$CI_COMMIT_SHA + image: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA services: - name: postgres:16 alias: postgres - - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA - alias: app - variables: - POSTGRES_URL: postgres://turniere:turniere@postgres:5432/turniere_blackbox - POSTGRES_USERNAME: turniere - POSTGRES_PASSWORD: turniere - SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod - MAILGUN_API_KEY: blackbox-test-api-key - MAILGUN_DOMAIN: blackbox.example.com - RAILS_LOG_TO_STDOUT: "1" - RAILS_SERVE_STATIC_FILES: "1" - command: - - bundle - - exec - - rails - - s - - -p - - "3000" - - -b - - 0.0.0.0 variables: POSTGRES_DB: turniere_blackbox POSTGRES_USER: turniere @@ -70,9 +50,9 @@ rails e2e spec: MAILGUN_DOMAIN: blackbox.example.com RAILS_LOG_TO_STDOUT: "1" RAILS_SERVE_STATIC_FILES: "1" - NO_PROXY: app,postgres,localhost,127.0.0.1 - no_proxy: app,postgres,localhost,127.0.0.1 - TURNIERE_E2E_BASE_URL: http://app:3000 + NO_PROXY: postgres,localhost,127.0.0.1 + no_proxy: postgres,localhost,127.0.0.1 + TURNIERE_E2E_BASE_URL: http://127.0.0.1:3000 TURNIERE_E2E_EMAIL: e2e@example.com TURNIERE_E2E_PASSWORD: password123 TURNIERE_E2E_USERNAME: e2e-user @@ -88,4 +68,4 @@ rails e2e spec: - when: always script: - cd /app - - inv blackbox-service + - inv blackbox-self diff --git a/AGENTS.md b/AGENTS.md index 7027e6b..a849811 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ Examples: - `inv test-shard --node-index=1 --node-total=8` - `inv lint` - `inv verify-http` +- `inv blackbox-self` - `inv blackbox-production` - `inv scenario-main-usecase` - `inv docker-build-all` @@ -23,7 +24,7 @@ Examples: - `inv docker-test-http-e2e` The GitLab CI pipeline is expected to use these tasks as well. -In CI, blackbox E2E should target sidecar service containers rather than Docker Compose inside the job. +In CI, blackbox E2E should run from the production image with Postgres as the sidecar and should avoid Docker Compose inside the job. The normal Rails spec suite is sharded through `inv test-shard ...` and should stay aligned with the CI matrix configuration. Useful GitLab push-option variables for saving CI minutes on public repos: diff --git a/script/verify_e2e_blackbox.rb b/script/verify_e2e_blackbox.rb new file mode 100644 index 0000000..acf91bf --- /dev/null +++ b/script/verify_e2e_blackbox.rb @@ -0,0 +1,65 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../e2e/lib/scenario_runner' + +def assert_equal(expected, actual, label) + return if expected == actual + + raise "#{label}: expected #{expected.inspect}, got #{actual.inspect}" +end + +def assert(condition, label) + return if condition + + raise label +end + +runner = TurniereE2E::ScenarioRunner.new( + base_url: ENV.fetch('TURNIERE_E2E_BASE_URL', 'http://127.0.0.1:3000'), + email: ENV.fetch('TURNIERE_E2E_EMAIL'), + password: ENV.fetch('TURNIERE_E2E_PASSWORD'), + username: ENV['TURNIERE_E2E_USERNAME'] +) + +puts 'Verifying main 4x4 group-stage lifecycle...' +main_result = runner.run_group_stage_main_usecase +assert_equal(8, main_result.dig(:checkpoints, :created, :playoff_teams_amount), 'main playoff_teams_amount') +assert_equal(8, main_result.dig(:checkpoints, :created, :instant_finalists_amount), 'main instant_finalists_amount') +assert_equal(0, main_result.dig(:checkpoints, :created, :intermediate_round_participants_amount), 'main intermediate_round_participants_amount') +assert_equal(4, main_result.dig(:checkpoints, :upcoming_group_matches, :match_ids).count, 'main upcoming_group_matches count') +assert_equal(16, main_result.dig(:checkpoints, :statistics, :group_scores_count), 'main group_scores_count') +main_playoff_entry_stage = main_result.dig(:checkpoints, :playoffs_created, :stage_summaries) + .select { |stage| stage[:level] != -1 } + .max_by { |stage| stage[:level] } +assert_equal(4, main_playoff_entry_stage[:match_states].count, 'main playoff entry match count') +main_final_attempt = main_result.dig(:checkpoints, :final_finish_attempted, :finish_attempt) +assert_equal(422, main_final_attempt[:status], 'main final finish status') +assert_equal('Moving Team one stage down failed', main_final_attempt.dig(:body, :error), 'main final finish error') + +puts 'Verifying 3-group intermediate-round lifecycle...' +intermediate_result = runner.run_group_stage_intermediate_round +intermediate_created = intermediate_result.fetch(:checkpoints).fetch(:created) +assert_equal(4, intermediate_created[:playoff_teams_amount], 'intermediate playoff_teams_amount') +assert_equal(3, intermediate_created[:instant_finalists_amount], 'intermediate instant_finalists_amount') +assert_equal(2, intermediate_created[:intermediate_round_participants_amount], 'intermediate intermediate_round_participants_amount') +intermediate_playoff_entry_stage = intermediate_result.dig(:checkpoints, :playoffs_created, :stage_summaries) + .select { |stage| stage[:level] != -1 } + .max_by { |stage| stage[:level] } +assert_equal('intermediate_stage', intermediate_playoff_entry_stage[:state], 'intermediate stage state') +assert_equal(3, intermediate_playoff_entry_stage[:match_states].count('single_team'), 'intermediate single_team count') +assert_equal(1, intermediate_playoff_entry_stage[:match_states].count('not_started'), 'intermediate not_started count') +intermediate_final_attempt = intermediate_result.dig(:checkpoints, :final_finish_attempted, :finish_attempt) +assert_equal(422, intermediate_final_attempt[:status], 'intermediate final finish status') +assert_equal('Moving Team one stage down failed', intermediate_final_attempt.dig(:body, :error), 'intermediate final finish error') + +[2, 3, 5, 8].each do |team_count| + puts "Verifying playoff-only lifecycle for #{team_count} teams..." + playoff_result = runner.run_playoff_only(team_count: team_count) + final_stage = playoff_result.dig(:checkpoints, :completed, :stage_summaries) + .find { |stage| stage[:level] == 0 } + assert(final_stage, "missing final stage for playoff-only #{team_count}") + assert_equal(['finished'], final_stage[:match_states], "playoff-only #{team_count} final stage") +end + +puts 'Blackbox verification passed.' diff --git a/tasks.py b/tasks.py index 1b71ac1..45926f3 100644 --- a/tasks.py +++ b/tasks.py @@ -252,11 +252,11 @@ def _run_blackbox_rspec(base_url, email, password, username): ) -def _start_test_server(): - log_handle = open("/tmp/turniere-e2e-server.log", "w") +def _start_server(command, env, log_path="/tmp/turniere-e2e-server.log"): + log_handle = open(log_path, "w") process = subprocess.Popen( - f"bundle exec rails s -e test -b {SERVER_HOST} -p {SERVER_PORT}", - shell=True, + ["/bin/sh", "-lc", command], + env=env, stdout=log_handle, stderr=subprocess.STDOUT, preexec_fn=os.setsid, @@ -264,6 +264,13 @@ def _start_test_server(): return process, log_handle +def _start_test_server(): + return _start_server( + f"bundle exec rails s -e test -b {SERVER_HOST} -p {SERVER_PORT}", + _env(), + ) + + def _stop_process(process, log_handle): try: os.killpg(os.getpgid(process.pid), signal.SIGTERM) @@ -646,6 +653,48 @@ def blackbox_service( _run_blackbox_rspec(base_url, email, password, username) +@task(name="blackbox-self") +def blackbox_self( + c, + base_url=TEST_BASE_URL, + email=E2E_EMAIL, + password=E2E_PASSWORD, + username=E2E_USERNAME, +): + """Run the pure-Ruby blackbox verifier inside the production image against a local production server.""" + env = _shared_production_env() + server_command = f"bundle exec rails s -b {SERVER_HOST} -p {SERVER_PORT}" + + _print_header("Running production database migrations") + c.run("bundle exec rails db:migrate", env=env, pty=True) + + process, log_handle = _start_server(server_command, env) + try: + _print_header(f"Waiting for app healthcheck at {base_url}") + _wait_for_http(base_url=base_url, timeout=120) + + _print_header("Bootstrapping confirmed E2E user") + c.run( + _bootstrap_user_command(email, password, username), + env=env, + pty=True, + ) + + _print_header("Running blackbox verifier") + c.run( + "ruby script/verify_e2e_blackbox.rb", + env=_env( + TURNIERE_E2E_BASE_URL=base_url, + TURNIERE_E2E_EMAIL=email, + TURNIERE_E2E_PASSWORD=password, + TURNIERE_E2E_USERNAME=username, + ), + pty=True, + ) + finally: + _stop_process(process, log_handle) + + @task(name="blackbox-production") def blackbox_production( c,