from invoke import task import os import re import shlex import signal import subprocess import time import urllib.request import urllib.error from urllib.parse import urlparse from pathlib import Path SERVER_HOST = "0.0.0.0" SERVER_PORT = "3000" TEST_BASE_URL = f"http://127.0.0.1:{SERVER_PORT}" BLACKBOX_BASE_URL = TEST_BASE_URL BLACKBOX_INTERNAL_BASE_URL = "http://app:3000" BLACKBOX_SOURCE_INTERNAL_BASE_URL = "http://source-app:3000" BLACKBOX_FOLLOWER_INTERNAL_BASE_URL = "http://follower-app:3000" BLACKBOX_SOURCE_HOST_PORT = "3002" BLACKBOX_FOLLOWER_HOST_PORT = "3003" E2E_EMAIL = "e2e@example.com" E2E_PASSWORD = "password123" E2E_USERNAME = "e2e-user" E2E_ALT_EMAIL = "e2e-alt@example.com" E2E_ALT_PASSWORD = "password123" E2E_ALT_USERNAME = "e2e-alt-user" PRODUCTION_TAG = "registry.gitlab.com/turniere/turniere-backend/production/commits:local" TEST_TAG = "registry.gitlab.com/turniere/turniere-backend/test/commits:local" DOCKERFILE_PATH = "Dockerfile" PRODUCTION_DOCKER_TARGET = "production" TEST_DOCKER_TARGET = "test" BLACKBOX_COMPOSE_FILE = "docker-compose.blackbox.yml" BLACKBOX_PROJECT = "turniere-blackbox" BLACKBOX_FOLLOW_COMPOSE_FILE = "docker-compose.blackbox-follow.yml" BLACKBOX_FOLLOW_PROJECT = "turniere-blackbox-follow" BLACKBOX_POSTGRES_IMAGE = "postgres:16" BLACKBOX_DB_NAME = "turniere_blackbox" BLACKBOX_SOURCE_DB_NAME = "turniere_source_blackbox" BLACKBOX_FOLLOWER_DB_NAME = "turniere_follower_blackbox" BLACKBOX_DB_USER = "turniere" BLACKBOX_DB_PASSWORD = "turniere" BLACKBOX_SECRET_KEY_BASE = "turniere-blackbox-secret-key-base-please-change-in-real-prod" BLACKBOX_MAILGUN_API_KEY = "blackbox-test-api-key" BLACKBOX_MAILGUN_DOMAIN = "blackbox.example.com" SPEC_ROOT = Path("spec") E2E_SPEC_ROOT = SPEC_ROOT / "e2e" / "http" ROOT = Path(__file__).resolve().parent DB_DIR = ROOT / "db" GEMFILE_LOCK_PATH = ROOT / "Gemfile.lock" BUNDLER_VERSION = GEMFILE_LOCK_PATH.read_text(encoding="utf-8").split("BUNDLED WITH", 1)[1].strip().splitlines()[-1].strip() MANUAL_TEST_DB_PATH = DB_DIR / "funky_manual_tournament_lab_do_not_delete.sqlite3" MANUAL_TEST_PORT = "3010" MANUAL_TEST_BASE_URL = f"http://127.0.0.1:{MANUAL_TEST_PORT}" def _env(**overrides): env = os.environ.copy() env.update({key: str(value) for key, value in overrides.items()}) return env def _rails_env(env=None): return str(env or os.environ.get("RAILS_ENV", "development")) def _sqlite_db_path(env=None): rails_env = _rails_env(env) if rails_env == "development": return DB_DIR / "development.sqlite3" if rails_env == "test": return DB_DIR / "test.sqlite3" return None def _delete_sqlite_db(env=None): db_path = _sqlite_db_path(env) if db_path is None: return None for suffix in ("", "-shm", "-wal", ".journal"): candidate = Path(f"{db_path}{suffix}") if candidate.exists(): candidate.unlink() return db_path def _bundle_command(*parts): return " ".join(["bundle", f"_{BUNDLER_VERSION}_", *parts]) def _bundle_exec(*parts): return _bundle_command("exec", *parts) def _wait_for_http(base_url=TEST_BASE_URL, timeout=60): deadline = time.time() + timeout healthz_url = f"{base_url}/healthz" opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) last_error = None while time.time() < deadline: try: with opener.open(healthz_url) as response: if response.status == 200: return last_error = f"unexpected status {response.status}" except urllib.error.HTTPError as error: body = error.read().decode("utf-8", errors="replace") last_error = f"http {error.code}: {body}" time.sleep(0.5) except Exception as error: last_error = repr(error) time.sleep(0.5) detail = f" last error: {last_error}" if last_error else "" raise RuntimeError(f"healthcheck did not become ready within {timeout}s: {healthz_url}.{detail}") def _docker_compose_cmd(project_name=BLACKBOX_PROJECT, compose_file=BLACKBOX_COMPOSE_FILE): return ["docker-compose", "-p", project_name, "-f", compose_file] def _run_subprocess(command, env=None, capture_output=False, check=True): completed = subprocess.run( command, env=env, text=True, capture_output=capture_output, check=False, ) if check and completed.returncode != 0: raise subprocess.CalledProcessError( completed.returncode, command, output=completed.stdout, stderr=completed.stderr, ) return completed def _stream_subprocess(command, env=None, check=True): process = subprocess.Popen( command, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, ) output_chunks = [] assert process.stdout is not None for line in process.stdout: print(line, end="", flush=True) output_chunks.append(line) process.wait() output = "".join(output_chunks) if check and process.returncode != 0: raise subprocess.CalledProcessError(process.returncode, command, output=output, stderr=None) return subprocess.CompletedProcess(command, process.returncode, stdout=output, stderr="") def _compose_env( host_port=SERVER_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, expect_profiling="true", ): return _env( TURNIERE_BLACKBOX_HOST_PORT=host_port, TURNIERE_BLACKBOX_APP_IMAGE=app_image, TURNIERE_BLACKBOX_RUNNER_IMAGE=runner_image, TURNIERE_BLACKBOX_POSTGRES_IMAGE=postgres_image, TURNIERE_BLACKBOX_POSTGRES_DB=BLACKBOX_DB_NAME, TURNIERE_BLACKBOX_POSTGRES_USER=BLACKBOX_DB_USER, TURNIERE_BLACKBOX_POSTGRES_PASSWORD=BLACKBOX_DB_PASSWORD, TURNIERE_BLACKBOX_SECRET_KEY_BASE=BLACKBOX_SECRET_KEY_BASE, TURNIERE_BLACKBOX_MAILGUN_API_KEY=BLACKBOX_MAILGUN_API_KEY, TURNIERE_BLACKBOX_MAILGUN_DOMAIN=BLACKBOX_MAILGUN_DOMAIN, TURNIERE_E2E_BASE_URL=BLACKBOX_INTERNAL_BASE_URL, TURNIERE_E2E_EXPECT_PROFILING=expect_profiling, TURNIERE_E2E_EMAIL=E2E_EMAIL, TURNIERE_E2E_PASSWORD=E2E_PASSWORD, TURNIERE_E2E_USERNAME=E2E_USERNAME, TURNIERE_E2E_ALT_EMAIL=E2E_ALT_EMAIL, TURNIERE_E2E_ALT_PASSWORD=E2E_ALT_PASSWORD, TURNIERE_E2E_ALT_USERNAME=E2E_ALT_USERNAME, ) def _follow_compose_env( source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): return _env( TURNIERE_BLACKBOX_SOURCE_HOST_PORT=source_host_port, TURNIERE_BLACKBOX_FOLLOWER_HOST_PORT=follower_host_port, TURNIERE_BLACKBOX_APP_IMAGE=app_image, TURNIERE_BLACKBOX_RUNNER_IMAGE=runner_image, TURNIERE_BLACKBOX_POSTGRES_IMAGE=postgres_image, TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB=BLACKBOX_SOURCE_DB_NAME, TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB=BLACKBOX_FOLLOWER_DB_NAME, TURNIERE_BLACKBOX_POSTGRES_USER=BLACKBOX_DB_USER, TURNIERE_BLACKBOX_POSTGRES_PASSWORD=BLACKBOX_DB_PASSWORD, TURNIERE_BLACKBOX_SECRET_KEY_BASE=BLACKBOX_SECRET_KEY_BASE, TURNIERE_BLACKBOX_MAILGUN_API_KEY=BLACKBOX_MAILGUN_API_KEY, TURNIERE_BLACKBOX_MAILGUN_DOMAIN=BLACKBOX_MAILGUN_DOMAIN, TURNIERE_SOURCE_E2E_BASE_URL=BLACKBOX_SOURCE_INTERNAL_BASE_URL, TURNIERE_FOLLOWER_E2E_BASE_URL=BLACKBOX_FOLLOWER_INTERNAL_BASE_URL, TURNIERE_E2E_EMAIL=E2E_EMAIL, TURNIERE_E2E_PASSWORD=E2E_PASSWORD, TURNIERE_E2E_USERNAME=E2E_USERNAME, TURNIERE_E2E_ALT_EMAIL=E2E_ALT_EMAIL, TURNIERE_E2E_ALT_PASSWORD=E2E_ALT_PASSWORD, TURNIERE_E2E_ALT_USERNAME=E2E_ALT_USERNAME, ) def _format_command(command): return " ".join(shlex.quote(part) for part in command) def _git_commit_sha(): try: return subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() except subprocess.CalledProcessError: return "unknown" def _docker_build_command(tag, target, platforms=None, push=False): if platforms and "," in str(platforms) and not push: raise ValueError("multi-platform buildx builds require push=True") if platforms or push: command = [ "docker", "buildx", "build", "--target", target, "--build-arg", f"GIT_COMMIT_SHA={_git_commit_sha()}", "-t", tag, "-f", DOCKERFILE_PATH, ] if platforms: command.extend(["--platform", platforms]) command.append("--push" if push else "--load") command.append(".") return _format_command(command) return _format_command([ "docker", "build", "--target", target, "--build-arg", f"GIT_COMMIT_SHA={_git_commit_sha()}", "-t", tag, "-f", DOCKERFILE_PATH, ".", ]) def _rspec_command(*paths): command = ["bundle", "exec", "rspec", *paths] if os.environ.get("CI"): command.extend(["--format", "documentation", "--no-color"]) else: command.extend(["--format", "progress", "--tty"]) return command def _compose_run(compose_args, env, capture_output=False, check=True, project_name=BLACKBOX_PROJECT, compose_file=BLACKBOX_COMPOSE_FILE): command = _docker_compose_cmd(project_name=project_name, compose_file=compose_file) + compose_args return _run_subprocess(command, env=env, capture_output=capture_output, check=check) def _print_header(title): print(f"\n== {title} ==", flush=True) def _blackbox_network_smoke(c, target_base_url): command = [ "ruby", "-rnet/http", "-ruri", "-e", ( "uri = URI.parse(ARGV.fetch(0) + '/healthz'); " "res = Net::HTTP.get_response(uri); " "abort(\"healthcheck #{res.code}\") unless res.is_a?(Net::HTTPSuccess); " "puts(\"network smoke ok #{uri}\")" ), target_base_url, ] c.run(_format_command(command), pty=True) def _blackbox_service_to_service_smoke(c, env, source_label, target_base_url): command = [ "bundle", "exec", "rails", "runner", ( "require 'net/http'; " "require 'uri'; " "uri = URI.parse(ENV.fetch('BLACKBOX_SMOKE_TARGET_URL') + '/healthz'); " "Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 3, read_timeout: 3) do |http| " "res = http.get(uri.request_uri); " "abort(\"service smoke #{res.code}\") unless res.is_a?(Net::HTTPSuccess); " "puts(\"service smoke ok #{ENV.fetch('BLACKBOX_SMOKE_SOURCE_LABEL')} -> #{uri}\"); " "end" ), ] smoke_env = env.copy() smoke_env.update( BLACKBOX_SMOKE_SOURCE_LABEL=source_label, BLACKBOX_SMOKE_TARGET_URL=target_base_url, ) c.run(_format_command(command), env=smoke_env, pty=True) def _bootstrap_user_command(email, password, username): return ( f"{_bundle_exec('rails', 'runner')} " f"\"user = User.find_or_initialize_by(email: '{email}'); " f"user.username = '{username}'; " f"user.password = '{password}'; " f"user.password_confirmation = '{password}'; " "user.confirmed_at = Time.current; " "user.uid = user.email; " "user.provider = 'email'; " "user.save!\"" ) def _manual_test_env(): return _env( RAILS_ENV="test", DATABASE_URL=f"sqlite3:{MANUAL_TEST_DB_PATH}", ) def _e2e_seed_scenarios(): return [ (E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME, "group_stage_main_usecase", "created"), (E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME, "group_stage_main_usecase", "final_finish_attempted"), (E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME, "group_stage_intermediate_round", "completed"), (E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME, "playoff_only", "completed"), (E2E_ALT_EMAIL, E2E_ALT_PASSWORD, E2E_ALT_USERNAME, "group_stage_main_usecase", "created"), (E2E_ALT_EMAIL, E2E_ALT_PASSWORD, E2E_ALT_USERNAME, "playoff_only", "completed"), ] def _seed_via_e2e_scenarios(c, base_url, base_env=None): base_env = base_env or {} for email, password, username, scenario, stop_at in _e2e_seed_scenarios(): _print_header(f"Seeding {scenario} as {email} until {stop_at}") c.run( f"ruby script/e2e_scenarios.rb {scenario} --stop-at {stop_at}", env=_env( **base_env, TURNIERE_E2E_BASE_URL=base_url, TURNIERE_E2E_EMAIL=email, TURNIERE_E2E_PASSWORD=password, TURNIERE_E2E_USERNAME=username, ), pty=True, ) def _parse_rspec_report(output): summary_match = re.search(r"(\d+)\s+examples?,\s+(\d+)\s+failures?(?:,\s+(\d+)\s+pending)?", output) failed_examples = re.findall(r"^\s*rspec\s+(.+)$", output, flags=re.MULTILINE) return { "examples": int(summary_match.group(1)) if summary_match else None, "failures": int(summary_match.group(2)) if summary_match else None, "pending": int(summary_match.group(3)) if summary_match and summary_match.group(3) else 0, "failed_examples": failed_examples, } def _print_rspec_report(report): examples = report["examples"] failures = report["failures"] pending = report["pending"] failed_examples = report["failed_examples"] if examples is None or failures is None: print("Rerun summary could not be parsed from the RSpec output.") return print(f"Examples: {examples}") print(f"Failures: {failures}") print(f"Pending: {pending}") if failed_examples: print("Failed examples:") for example in failed_examples: print(f"- {example}") def _print_service_logs(env, service, tail=120): _print_header(f"{service} logs (last {tail} lines)") logs = _compose_run(["logs", "--tail", str(tail), service], env=env, capture_output=True, check=False) output = logs.stdout or logs.stderr or "" print(output.rstrip()) def _spec_files(): return sorted( path for path in SPEC_ROOT.rglob("*_spec.rb") if not path.is_relative_to(E2E_SPEC_ROOT) ) def _spec_weight(path): return sum(1 for _ in path.open("r", encoding="utf-8")) def _spec_shards(node_total): if node_total <= 0: raise ValueError("node_total must be positive") shards = [{"weight": 0, "files": []} for _ in range(node_total)] weighted_files = sorted( ((path, _spec_weight(path)) for path in _spec_files()), key=lambda item: (-item[1], str(item[0])), ) for path, weight in weighted_files: shard = min(shards, key=lambda item: (item["weight"], len(item["files"]))) shard["files"].append(path) shard["weight"] += weight return shards def _shared_production_env(): return _env( RAILS_ENV="production", POSTGRES_URL=os.environ.get( "POSTGRES_URL", f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@postgres:5432/{BLACKBOX_DB_NAME}", ), POSTGRES_USERNAME=os.environ.get("POSTGRES_USERNAME", BLACKBOX_DB_USER), POSTGRES_PASSWORD=os.environ.get("POSTGRES_PASSWORD", BLACKBOX_DB_PASSWORD), SECRET_KEY_BASE=os.environ.get("SECRET_KEY_BASE", BLACKBOX_SECRET_KEY_BASE), MAILGUN_API_KEY=os.environ.get("MAILGUN_API_KEY", BLACKBOX_MAILGUN_API_KEY), MAILGUN_DOMAIN=os.environ.get("MAILGUN_DOMAIN", BLACKBOX_MAILGUN_DOMAIN), RAILS_LOG_TO_STDOUT=os.environ.get("RAILS_LOG_TO_STDOUT", "1"), RAILS_SERVE_STATIC_FILES=os.environ.get("RAILS_SERVE_STATIC_FILES", "1"), ) def _production_env_for(db_host, db_name): return _env( RAILS_ENV="production", POSTGRES_URL=f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@{db_host}:5432/{db_name}", POSTGRES_USERNAME=BLACKBOX_DB_USER, POSTGRES_PASSWORD=BLACKBOX_DB_PASSWORD, SECRET_KEY_BASE=BLACKBOX_SECRET_KEY_BASE, MAILGUN_API_KEY=BLACKBOX_MAILGUN_API_KEY, MAILGUN_DOMAIN=BLACKBOX_MAILGUN_DOMAIN, RAILS_LOG_TO_STDOUT="1", RAILS_SERVE_STATIC_FILES="1", ) def _run_blackbox_rspec(base_url, email, password, username, alt_email=E2E_ALT_EMAIL, alt_password=E2E_ALT_PASSWORD, alt_username=E2E_ALT_USERNAME, spec_path="spec/e2e/http", extra_env=None): env = _env( TURNIERE_E2E_BASE_URL=base_url, TURNIERE_E2E_EXPECT_PROFILING="false", TURNIERE_E2E_EMAIL=email, TURNIERE_E2E_PASSWORD=password, TURNIERE_E2E_USERNAME=username, TURNIERE_E2E_ALT_EMAIL=alt_email, TURNIERE_E2E_ALT_PASSWORD=alt_password, TURNIERE_E2E_ALT_USERNAME=alt_username, ) if extra_env: env.update(extra_env) spec_paths = spec_path if isinstance(spec_path, (list, tuple)) else [spec_path] command = _rspec_command(*spec_paths) print(_format_command(command), flush=True) result = _stream_subprocess(command, env=env, check=False) report = _parse_rspec_report(f"{result.stdout}\n{result.stderr}") _print_header("Blackbox report") _print_rspec_report(report) if result.returncode != 0: raise subprocess.CalledProcessError( result.returncode, result.args, output=result.stdout, stderr=result.stderr, ) def _start_server(command, env, log_path="/tmp/turniere-e2e-server.log"): log_handle = open(log_path, "w") process = subprocess.Popen( ["/bin/sh", "-lc", command], env=env, stdout=log_handle, stderr=subprocess.STDOUT, preexec_fn=os.setsid, ) return process, log_handle def _start_test_server(host=SERVER_HOST, port=SERVER_PORT): return _start_server( f"{_bundle_exec('rails', 's')} -e test -b {host} -p {port}", _env(), ) def _stop_process(process, log_handle): try: os.killpg(os.getpgid(process.pid), signal.SIGTERM) except ProcessLookupError: pass finally: log_handle.close() def _bootstrap_blackbox_users(c, env, email, password, username, alt_email, alt_password, alt_username): c.run( _bootstrap_user_command(email, password, username), env=env, pty=True, ) c.run( _bootstrap_user_command(alt_email, alt_password, alt_username), env=env, pty=True, ) @task def setup(c): """Install the pinned Bundler and local backend gem dependencies.""" install_bundler = c.run( f"gem list -i bundler -v {BUNDLER_VERSION}", warn=True, hide=True, ) if not install_bundler.ok: c.run(f"gem install bundler:{BUNDLER_VERSION}", pty=True) c.run(f"{_bundle_command('config', 'set', 'without', 'production')}", pty=True) c.run(f"{_bundle_command('config', 'set', 'with', 'development test')}", pty=True) c.run(_bundle_command("install"), pty=True) @task(name="db-migrate") def db_migrate(c, env=None): """Run database migrations locally.""" c.run(f"RAILS_ENV={_rails_env(env)} {_bundle_exec('rails', 'db:migrate')}", pty=True) @task(name="db-reset") def db_reset(c, env="test"): """Delete the local SQLite database for the requested non-production Rails env.""" rails_env = _rails_env(env) if rails_env == "production": raise ValueError("db-reset refuses to run in production") deleted_path = _delete_sqlite_db(rails_env) if deleted_path is None: print(f"No local SQLite database configured for RAILS_ENV={rails_env}. Nothing deleted.") return print(f"Deleted {deleted_path}") @task def server(c, host=SERVER_HOST, port=SERVER_PORT, env=None): """Start the backend locally in development.""" c.run(f"RAILS_ENV={_rails_env(env)} {_bundle_exec('rails', 's')} -b {host} -p {port}", pty=True) @task(name="start-local") def start_local(c, host=SERVER_HOST, port=SERVER_PORT, bootstrap_user=False): """Install local deps if needed, migrate the dev database, and start the server.""" setup(c) db_migrate(c, env="development") if bootstrap_user: bootstrap_dev_user(c) server(c, host=host, port=port, env="development") @task(name="server-test") def server_test(c, host=SERVER_HOST, port=SERVER_PORT): """Start the backend locally in test mode.""" server(c, host=host, port=port, env="test") @task(name="start-test-local") def start_test_local( c, host=SERVER_HOST, port=SERVER_PORT, bootstrap_user=True, reset_db=True, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, alt_email=E2E_ALT_EMAIL, alt_password=E2E_ALT_PASSWORD, alt_username=E2E_ALT_USERNAME, ): """Install deps, recreate the local test DB, bootstrap HTTP E2E users, and start the test server.""" setup(c) if reset_db: db_reset(c, env="test") db_migrate(c, env="test") if bootstrap_user: bootstrap_e2e_user(c, email=email, password=password, username=username) bootstrap_e2e_user(c, email=alt_email, password=alt_password, username=alt_username) server(c, host=host, port=port, env="test") @task def test(c): """Run the normal RSpec suite.""" _stream_subprocess(_rspec_command(), env=_env()) @task(name="test-shard") def test_shard(c, node_index=1, node_total=8, dry_run=False): """Run one deterministic shard of the non-E2E RSpec suite.""" node_index = int(node_index) node_total = int(node_total) if node_index < 1 or node_index > node_total: raise ValueError(f"node_index must be between 1 and {node_total}, got {node_index}") shards = _spec_shards(node_total) shard = shards[node_index - 1] files = [str(path) for path in shard["files"]] _print_header(f"RSpec shard {node_index}/{node_total}") print(f"Files: {len(files)}") print(f"Estimated weight: {shard['weight']}") if files: print("Included specs:") for path in files: print(f"- {path}") else: print("No spec files assigned to this shard.") if dry_run: return if not files: return _stream_subprocess( _rspec_command("--require", "./spec/rails_helper", *files), env=_env(), ) @task def lint(c): """Run RuboCop.""" c.run(_bundle_exec("rubocop"), pty=True) @task(name="bootstrap-dev-user") def bootstrap_dev_user(c, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME): """Create or update a confirmed local development user.""" command = ( f"RAILS_ENV=development {_bundle_exec('rails', 'runner')} " f"\"user = User.find_or_initialize_by(email: '{email}'); " f"user.username = '{username}'; " f"user.password = '{password}'; " f"user.password_confirmation = '{password}'; " "user.confirmed_at = Time.current; " "user.uid = user.email; " "user.provider = 'email'; " "user.save!\"" ) c.run(command, pty=True) @task(name="bootstrap-e2e-user") def bootstrap_e2e_user(c, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME): """Create or update the confirmed HTTP E2E user in test env.""" command = ( f"RAILS_ENV=test {_bundle_exec('rails', 'runner')} " f"\"user = User.find_or_initialize_by(email: '{email}'); " f"user.username = '{username}'; " f"user.password = '{password}'; " f"user.password_confirmation = '{password}'; " "user.confirmed_at = Time.current; " "user.uid = user.email; " "user.provider = 'email'; " "user.save!\"" ) c.run(command, pty=True) @task(name="test-http-e2e") def test_http_e2e( c, base_url=TEST_BASE_URL, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, alt_email=E2E_ALT_EMAIL, alt_password=E2E_ALT_PASSWORD, alt_username=E2E_ALT_USERNAME, spec="", ): """Run HTTP E2E specs against an already running backend.""" command = f"{_bundle_exec('rspec')} spec/e2e/http" if spec: command = f"{_bundle_exec('rspec')} {shlex.quote(spec)}" c.run( command, env=_env( TURNIERE_E2E_BASE_URL=base_url, TURNIERE_E2E_EMAIL=email, TURNIERE_E2E_PASSWORD=password, TURNIERE_E2E_USERNAME=username, TURNIERE_E2E_ALT_EMAIL=alt_email, TURNIERE_E2E_ALT_PASSWORD=alt_password, TURNIERE_E2E_ALT_USERNAME=alt_username, ), pty=True, ) @task(name="verify-http") def verify_http( c, base_url=TEST_BASE_URL, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, alt_email=E2E_ALT_EMAIL, alt_password=E2E_ALT_PASSWORD, alt_username=E2E_ALT_USERNAME, reset_db=True, spec="", ): """Boot a local test server, then run the HTTP E2E suite against it.""" if reset_db: db_reset(c, env="test") db_migrate(c, env="test") bootstrap_e2e_user(c, email=email, password=password, username=username) bootstrap_e2e_user(c, email=alt_email, password=alt_password, username=alt_username) parsed_base_url = urlparse(base_url) server_host = parsed_base_url.hostname or "127.0.0.1" server_port = parsed_base_url.port or 3000 process, log_handle = _start_test_server(host=server_host, port=server_port) try: _wait_for_http(base_url=base_url, timeout=60) test_http_e2e( c, base_url=base_url, email=email, password=password, username=username, alt_email=alt_email, alt_password=alt_password, alt_username=alt_username, spec=spec, ) finally: _stop_process(process, log_handle) @task def verify(c): """Run lint, normal specs, and HTTP E2E locally.""" lint(c) test(c) verify_http(c) @task(name="scenario-main-usecase") def scenario_main_usecase( c, base_url=TEST_BASE_URL, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, stop_at="final_finish_attempted", ): """Materialize the main 4x4 group-stage scenario over HTTP.""" c.run( f"ruby script/e2e_scenarios.rb group_stage_main_usecase --stop-at {stop_at}", env=_env( TURNIERE_E2E_BASE_URL=base_url, TURNIERE_E2E_EMAIL=email, TURNIERE_E2E_PASSWORD=password, TURNIERE_E2E_USERNAME=username, ), pty=True, ) @task(name="scenario-intermediate") def scenario_intermediate( c, base_url=TEST_BASE_URL, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, stop_at="final_finish_attempted", ): """Materialize the 3-group intermediate-round scenario over HTTP.""" c.run( f"ruby script/e2e_scenarios.rb group_stage_intermediate_round --stop-at {stop_at}", env=_env( TURNIERE_E2E_BASE_URL=base_url, TURNIERE_E2E_EMAIL=email, TURNIERE_E2E_PASSWORD=password, TURNIERE_E2E_USERNAME=username, ), pty=True, ) @task(name="scenario-playoff") def scenario_playoff( c, team_count=8, base_url=TEST_BASE_URL, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, stop_at="completed", ): """Materialize a playoff-only scenario over HTTP.""" c.run( f"ruby script/e2e_scenarios.rb playoff_only --team-count {team_count} --stop-at {stop_at}", env=_env( TURNIERE_E2E_BASE_URL=base_url, TURNIERE_E2E_EMAIL=email, TURNIERE_E2E_PASSWORD=password, TURNIERE_E2E_USERNAME=username, ), pty=True, ) @task(name="scenario-render-profile") def scenario_render_profile( c, group_count=8, teams_per_group=4, playoff_teams_amount=8, base_url=TEST_BASE_URL, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, stop_at="profiled_show", ): """Materialize a group-stage tournament over HTTP and print tournament show profiling.""" c.run( "ruby script/e2e_scenarios.rb group_stage_render_profile " f"--group-count {group_count} " f"--teams-per-group {teams_per_group} " f"--playoff-teams-amount {playoff_teams_amount} " f"--stop-at {stop_at}", env=_env( TURNIERE_E2E_BASE_URL=base_url, TURNIERE_E2E_EMAIL=email, TURNIERE_E2E_PASSWORD=password, TURNIERE_E2E_USERNAME=username, ), pty=True, ) @task(name="docker-build-production") def docker_build_production(c, tag=PRODUCTION_TAG, platforms=None, push=False): """Build the production Docker image.""" c.run(_docker_build_command(tag, PRODUCTION_DOCKER_TARGET, platforms=platforms, push=push), pty=True) @task(name="docker-build-test") def docker_build_test(c, tag=TEST_TAG, platforms=None, push=False): """Build the test Docker image.""" c.run(_docker_build_command(tag, TEST_DOCKER_TARGET, platforms=platforms, push=push), pty=True) @task(name="docker-build-all") def docker_build_all(c): """Build the production and test Docker images.""" docker_build_production(c) docker_build_test(c) @task(name="docker-blackbox-up") def docker_blackbox_up( c, host_port=SERVER_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, build=True, ): """Build the Docker images, then boot Postgres and the production app for blackbox testing.""" if build: docker_build_all(c) env = _compose_env( host_port=host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, expect_profiling="false", ) _print_header("Starting production blackbox stack") _compose_run(["down", "-v", "--remove-orphans"], env=env, check=False) _compose_run(["up", "-d", "postgres", "app"], env=env) base_url = f"http://127.0.0.1:{host_port}" _print_header(f"Waiting for app healthcheck at {base_url}") _wait_for_http(base_url=base_url, timeout=120) _print_header("Bootstrapping confirmed E2E user") for bootstrap_email, bootstrap_password, bootstrap_username in ( (E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME), (E2E_ALT_EMAIL, E2E_ALT_PASSWORD, E2E_ALT_USERNAME), ): bootstrap = _compose_run( ["exec", "-T", "app", "bash", "-lc", _bootstrap_user_command(bootstrap_email, bootstrap_password, bootstrap_username)], env=env, capture_output=True, check=False, ) if bootstrap.returncode != 0: print(bootstrap.stdout or "", end="") print(bootstrap.stderr or "", end="") raise subprocess.CalledProcessError( bootstrap.returncode, bootstrap.args, output=bootstrap.stdout, stderr=bootstrap.stderr, ) print("Production app is ready for blackbox E2E.") @task(name="docker-blackbox-down") def docker_blackbox_down( c, host_port=SERVER_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Stop and remove the production blackbox stack.""" env = _compose_env( host_port=host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) _compose_run(["down", "-v", "--remove-orphans"], env=env, check=False) @task(name="docker-blackbox-test") def docker_blackbox_test( c, host_port=SERVER_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Run the HTTP E2E suite against the running production blackbox stack.""" env = _compose_env( host_port=host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, expect_profiling="false", ) _print_header("Running HTTP E2E against production image") command = [ "run", "--rm", "e2e", "bundle", "exec", "rspec", "spec/e2e/http", "--format", "documentation", ] print(_format_command(_docker_compose_cmd() + command)) result = _compose_run(command, env=env, capture_output=True, check=False) if result.stdout: print(result.stdout.rstrip()) if result.stderr: print(result.stderr.rstrip()) report = _parse_rspec_report(f"{result.stdout}\n{result.stderr}") _print_header("Blackbox report") _print_rspec_report(report) if result.returncode != 0: raise subprocess.CalledProcessError( result.returncode, result.args, output=result.stdout, stderr=result.stderr, ) @task(name="docker-blackbox-follow-up") def docker_blackbox_follow_up( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, build=True, ): """Build Docker images, then boot source/follower Postgres and production apps for follower blackbox testing.""" if build: docker_build_all(c) env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) _print_header("Starting follower production blackbox stack") _compose_run( ["down", "-v", "--remove-orphans"], env=env, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) _compose_run( ["up", "-d", "source-postgres", "follower-postgres", "source-app", "follower-app"], env=env, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) source_base_url = f"http://127.0.0.1:{source_host_port}" follower_base_url = f"http://127.0.0.1:{follower_host_port}" _print_header(f"Waiting for source app healthcheck at {source_base_url}") _wait_for_http(base_url=source_base_url, timeout=120) _print_header(f"Waiting for follower app healthcheck at {follower_base_url}") _wait_for_http(base_url=follower_base_url, timeout=120) print("Follower production apps ready for blackbox E2E.") @task(name="docker-blackbox-follow-manual-up") def docker_blackbox_follow_manual_up( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, build=True, ): """Boot local leader/follower production setup for manual testing and print next steps.""" docker_blackbox_follow_up( c, source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, build=build, ) env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) _print_header("Running source production database migrations") _compose_run( ["exec", "-T", "source-app", "bash", "-lc", "bundle exec rails db:migrate"], env=env, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) _print_header("Running follower production database migrations") _compose_run( ["exec", "-T", "follower-app", "bash", "-lc", "bundle exec rails db:migrate"], env=env, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) _print_header("Bootstrapping confirmed E2E users on source and follower") docker_blackbox_follow_bootstrap_users( c, source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) source_base_url = f"http://127.0.0.1:{source_host_port}" follower_base_url = f"http://127.0.0.1:{follower_host_port}" _print_header("Leader/follower manual setup ready") print(f"Leader URL: {source_base_url}") print(f"Follower URL: {follower_base_url}") print(f"E2E owner email: {E2E_EMAIL}") print(f"E2E owner password: {E2E_PASSWORD}") print("Next steps:") print("1. Create read-only follower tournament on follower backend.") print("2. Create normal leader tournament on leader backend.") print("3. Configure leader sync_target_url to follower /sync_state endpoint with shared token.") print("4. Point frontend local leader view at leader URL and remote view at follower URL.") print("5. Tear down with: inv docker-blackbox-follow-down") @task(name="docker-blackbox-follow-down") def docker_blackbox_follow_down( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Stop and remove follower production blackbox stack.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) _compose_run( ["down", "-v", "--remove-orphans"], env=env, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) @task(name="docker-blackbox-follow-test") def docker_blackbox_follow_test( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Run follower-sync HTTP E2E suite against running source/follower production blackbox stack.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) _print_header("Running follower sync HTTP E2E against production images") command = [ "run", "--rm", "e2e", "inv", "blackbox-follow-service", ] full_command = _docker_compose_cmd(project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE) + command print(_format_command(full_command), flush=True) result = _stream_subprocess(full_command, env=env, check=False) if result.returncode != 0: raise subprocess.CalledProcessError( result.returncode, result.args, output=result.stdout, stderr=result.stderr, ) @task(name="docker-blackbox-follow-bootstrap-users") def docker_blackbox_follow_bootstrap_users( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Bootstrap confirmed E2E users in local source/follower production blackbox DBs.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) for app_service in ("source-app", "follower-app"): _print_header(f"Bootstrapping confirmed E2E users in {app_service}") for bootstrap_email, bootstrap_password, bootstrap_username in ( (E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME), (E2E_ALT_EMAIL, E2E_ALT_PASSWORD, E2E_ALT_USERNAME), ): bootstrap = _compose_run( ["exec", "-T", app_service, "bash", "-lc", _bootstrap_user_command(bootstrap_email, bootstrap_password, bootstrap_username)], env=env, capture_output=True, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) if bootstrap.returncode != 0: print(bootstrap.stdout or "", end="") print(bootstrap.stderr or "", end="") raise subprocess.CalledProcessError( bootstrap.returncode, bootstrap.args, output=bootstrap.stdout, stderr=bootstrap.stderr, ) @task(name="docker-blackbox-follow-debug-auth") def docker_blackbox_follow_debug_auth( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Debug source/follower login and first authenticated create inside local follower blackbox runner.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) script = """ require_relative 'e2e/lib/api_client' def inspect_client(base_url) client = TurniereE2E::ApiClient.new(base_url: base_url) login = client.login!(email: ENV.fetch('TURNIERE_E2E_EMAIL'), password: ENV.fetch('TURNIERE_E2E_PASSWORD')) puts({ base_url: base_url, login_status: login[:status], login_headers: login[:headers].slice('access-token', 'client', 'uid', 'expiry', 'token-type', 'authorization'), authenticated: client.authenticated? }.inspect) create = client.post('/tournaments', body: { name: "Debug #{base_url}", description: 'Debug tournament', public: false, teams: [ { name: 'One' }, { name: 'Two' }, { name: 'Three' }, { name: 'Four' } ] }) puts({ base_url: base_url, create_status: create[:status], create_json: create[:json], create_headers: create[:headers].slice('access-token', 'client', 'uid', 'expiry', 'token-type', 'authorization') }.inspect) end inspect_client(ENV.fetch('TURNIERE_SOURCE_E2E_BASE_URL')) inspect_client(ENV.fetch('TURNIERE_FOLLOWER_E2E_BASE_URL')) """ command = [ "run", "--rm", "e2e", "ruby", "-e", script, ] print(_format_command(_docker_compose_cmd(project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE) + command)) result = _compose_run( command, env=env, capture_output=True, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) if result.stdout: print(result.stdout.rstrip()) if result.stderr: print(result.stderr.rstrip()) if result.returncode != 0: raise subprocess.CalledProcessError( result.returncode, result.args, output=result.stdout, stderr=result.stderr, ) @task(name="docker-blackbox-follow-debug-seed") def docker_blackbox_follow_debug_seed( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Debug repeated mixed tournament creation inside local follower blackbox runner.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) script = """ require_relative 'e2e/lib/api_client' def create_playoff_payload(name) { name: name, description: 'Debug playoff tournament', public: false, teams: [ { name: 'One' }, { name: 'Two' }, { name: 'Three' }, { name: 'Four' } ] } end def create_group_payload(name) { name: name, description: 'Debug group tournament', public: true, group_stage: true, playoff_teams_amount: 8, teams: 4.times.flat_map do |group_index| 4.times.map do |seed_index| { name: "#{name}-G#{group_index + 1}-S#{seed_index + 1}", group: group_index } end end } end def inspect_seed(base_url) client = TurniereE2E::ApiClient.new(base_url: base_url) login = client.login!(email: ENV.fetch('TURNIERE_E2E_EMAIL'), password: ENV.fetch('TURNIERE_E2E_PASSWORD')) puts({ base_url: base_url, login_status: login[:status], authenticated: client.authenticated? }.inspect) [ ['playoff-1', create_playoff_payload('Debug Playoff 1')], ['group-1', create_group_payload('Debug Group 1')], ['playoff-2', create_playoff_payload('Debug Playoff 2')], ['group-2', create_group_payload('Debug Group 2')] ].each do |label, payload| response = client.post('/tournaments', body: payload) puts({ base_url: base_url, step: label, status: response[:status], json: response[:json], headers: response[:headers].slice('access-token', 'client', 'uid', 'expiry', 'token-type', 'authorization'), authenticated: client.authenticated? }.inspect) end end inspect_seed(ENV.fetch('TURNIERE_SOURCE_E2E_BASE_URL')) inspect_seed(ENV.fetch('TURNIERE_FOLLOWER_E2E_BASE_URL')) """ command = [ "run", "--rm", "e2e", "ruby", "-e", script, ] print(_format_command(_docker_compose_cmd(project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE) + command)) result = _compose_run( command, env=env, capture_output=True, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) if result.stdout: print(result.stdout.rstrip()) if result.stderr: print(result.stderr.rstrip()) if result.returncode != 0: raise subprocess.CalledProcessError( result.returncode, result.args, output=result.stdout, stderr=result.stderr, ) @task(name="docker-blackbox-follow-debug-group-create") def docker_blackbox_follow_debug_group_create( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Debug single fresh-login group-stage tournament creation inside local follower blackbox runner.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) script = """ require_relative 'e2e/lib/api_client' def create_group_payload(name) { name: name, description: 'Debug group tournament', public: true, group_stage: true, playoff_teams_amount: 8, teams: 4.times.flat_map do |group_index| 4.times.map do |seed_index| { name: "#{name}-G#{group_index + 1}-S#{seed_index + 1}", group: group_index } end end } end def inspect_group_create(base_url) client = TurniereE2E::ApiClient.new(base_url: base_url) login = client.login!(email: ENV.fetch('TURNIERE_E2E_EMAIL'), password: ENV.fetch('TURNIERE_E2E_PASSWORD')) puts({ base_url: base_url, login_status: login[:status], authenticated: client.authenticated? }.inspect) response = client.post('/tournaments', body: create_group_payload('Fresh Debug Group')) puts({ base_url: base_url, create_status: response[:status], create_json: response[:json], create_headers: response[:headers].slice('access-token', 'client', 'uid', 'expiry', 'token-type', 'authorization') }.inspect) end inspect_group_create(ENV.fetch('TURNIERE_SOURCE_E2E_BASE_URL')) inspect_group_create(ENV.fetch('TURNIERE_FOLLOWER_E2E_BASE_URL')) """ command = [ "run", "--rm", "e2e", "ruby", "-e", script, ] print(_format_command(_docker_compose_cmd(project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE) + command)) result = _compose_run( command, env=env, capture_output=True, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) if result.stdout: print(result.stdout.rstrip()) if result.stderr: print(result.stderr.rstrip()) if result.returncode != 0: raise subprocess.CalledProcessError( result.returncode, result.args, output=result.stdout, stderr=result.stderr, ) @task(name="docker-blackbox-follow-debug-sync") def docker_blackbox_follow_debug_sync( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, log_tail=120, ): """Print local follower blackbox sync queue state and recent app logs.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) script = """ puts '== Source tournaments ==' Tournament.order(:id).pluck(:id, :name, :read_only_mode, :sync_target_url, :sync_last_pushed_at, :sync_last_push_error).each { |row| p row } puts '== Source sync queue ==' TournamentSyncQueueEntry.order(:id).pluck(:id, :tournament_id, :status, :attempts, :last_error, :next_attempt_at, :last_attempt_at).each { |row| p row } puts '== Follower tournaments ==' Tournament.order(:id).pluck(:id, :name, :read_only_mode, :sync_target_url, :sync_last_pushed_at, :sync_last_push_error).each { |row| p row } """ for service, db_host in (("source-app", "source-postgres"), ("follower-app", "follower-postgres")): _print_header(f"Sync state from {service}") db_name = BLACKBOX_SOURCE_DB_NAME if service == "source-app" else BLACKBOX_FOLLOWER_DB_NAME service_env = _production_env_for(db_host, db_name) runner = _compose_run( ["run", "--rm", "-e", f"POSTGRES_URL={service_env['POSTGRES_URL']}", "-e", f"POSTGRES_USERNAME={service_env['POSTGRES_USERNAME']}", "-e", f"POSTGRES_PASSWORD={service_env['POSTGRES_PASSWORD']}", "-e", f"SECRET_KEY_BASE={service_env['SECRET_KEY_BASE']}", "-e", f"MAILGUN_API_KEY={service_env['MAILGUN_API_KEY']}", "-e", f"MAILGUN_DOMAIN={service_env['MAILGUN_DOMAIN']}", "-e", f"RAILS_LOG_TO_STDOUT={service_env['RAILS_LOG_TO_STDOUT']}", "-e", f"RAILS_SERVE_STATIC_FILES={service_env['RAILS_SERVE_STATIC_FILES']}", "-e", "RAILS_ENV=production", "e2e", "bundle", "exec", "rails", "runner", script], env=env, capture_output=True, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) if runner.stdout: print(runner.stdout.rstrip()) if runner.stderr: print(runner.stderr.rstrip()) if runner.returncode != 0: raise subprocess.CalledProcessError( runner.returncode, runner.args, output=runner.stdout, stderr=runner.stderr, ) _print_header(f"Recent logs from {service}") logs = _compose_run( ["logs", "--tail", str(log_tail), service], env=env, capture_output=True, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) output = logs.stdout or logs.stderr or "" print(output.rstrip()) @task(name="docker-blackbox-follow-debug-compare") def docker_blackbox_follow_debug_compare( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Compare latest leader tournament and follower read-only tournament via HTTP API.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) script = """ def signature(tournament) { id: tournament.id, name: tournament.name, public: tournament.public, playoff_teams_amount: tournament.playoff_teams_amount, instant_finalists_amount: tournament.instant_finalists_amount, intermediate_round_participants_amount: tournament.intermediate_round_participants_amount, sync_source_tournament_id: tournament.respond_to?(:sync_source_tournament_id) ? tournament.sync_source_tournament_id : nil, team_names: tournament.teams.order(:id).pluck(:name), stage_levels: tournament.stages.order(:level).map { |stage| [stage.level, stage.state, stage.groups.count, stage.matches.count] }, total_matches: tournament.matches.count, decider_matches: tournament.matches.count(&:decider?) } end tournament = if ENV['FOLLOW_COMPARE_MODE'] == 'leader' Tournament.where.not(sync_target_url: nil).order(:id).last else Tournament.where(read_only_mode: true).order(:id).last end puts signature(tournament).inspect """ for mode, db_host, db_name in (("leader", "source-postgres", BLACKBOX_SOURCE_DB_NAME), ("follower", "follower-postgres", BLACKBOX_FOLLOWER_DB_NAME)): _print_header(f"Compare {mode} tournament") service_env = _production_env_for(db_host, db_name) runner = _compose_run( ["run", "--rm", "-e", f"POSTGRES_URL={service_env['POSTGRES_URL']}", "-e", f"POSTGRES_USERNAME={service_env['POSTGRES_USERNAME']}", "-e", f"POSTGRES_PASSWORD={service_env['POSTGRES_PASSWORD']}", "-e", f"SECRET_KEY_BASE={service_env['SECRET_KEY_BASE']}", "-e", f"MAILGUN_API_KEY={service_env['MAILGUN_API_KEY']}", "-e", f"MAILGUN_DOMAIN={service_env['MAILGUN_DOMAIN']}", "-e", f"RAILS_LOG_TO_STDOUT={service_env['RAILS_LOG_TO_STDOUT']}", "-e", f"RAILS_SERVE_STATIC_FILES={service_env['RAILS_SERVE_STATIC_FILES']}", "-e", "RAILS_ENV=production", "-e", f"FOLLOW_COMPARE_MODE={mode}", "e2e", "bundle", "exec", "rails", "runner", script], env=env, capture_output=True, check=False, project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) if runner.stdout: print(runner.stdout.rstrip()) if runner.stderr: print(runner.stderr.rstrip()) if runner.returncode != 0: raise subprocess.CalledProcessError( runner.returncode, runner.args, output=runner.stdout, stderr=runner.stderr, ) @task(name="docker-blackbox-follow-run") def docker_blackbox_follow_run( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, build=True, keep_running=False, ): """Build, boot, run follower-sync blackbox E2E, then tear down unless keep_running=true.""" exit_error = None try: docker_blackbox_follow_up( c, source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, build=build, ) docker_blackbox_follow_test( c, source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) except subprocess.CalledProcessError as error: exit_error = error raise finally: if keep_running: print("Keeping follower production blackbox stack running.") else: docker_blackbox_follow_down( c, source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) if exit_error is not None: print("Follower blackbox run failed.") @task(name="blackbox-service") def blackbox_service( c, base_url=BLACKBOX_INTERNAL_BASE_URL, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, alt_email=E2E_ALT_EMAIL, alt_password=E2E_ALT_PASSWORD, alt_username=E2E_ALT_USERNAME, ): """Run blackbox HTTP E2E against an already running production app plus Postgres sidecars.""" _print_header("Running production database migrations") c.run("bundle exec rails db:migrate", env=_shared_production_env(), pty=True) _print_header(f"Waiting for app healthcheck at {base_url}") _wait_for_http(base_url=base_url, timeout=120) _print_header("Bootstrapping confirmed E2E user") _bootstrap_blackbox_users(c, _shared_production_env(), email, password, username, alt_email, alt_password, alt_username) _print_header("Running HTTP E2E against production image") _run_blackbox_rspec(base_url, email, password, username, alt_email, alt_password, alt_username) @task(name="blackbox-follow-service") def blackbox_follow_service( c, source_base_url=BLACKBOX_SOURCE_INTERNAL_BASE_URL, follower_base_url=BLACKBOX_FOLLOWER_INTERNAL_BASE_URL, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, alt_email=E2E_ALT_EMAIL, alt_password=E2E_ALT_PASSWORD, alt_username=E2E_ALT_USERNAME, ): """Run follower-sync HTTP E2E against two independent production app services.""" source_env = _production_env_for("source-postgres", BLACKBOX_SOURCE_DB_NAME) follower_env = _production_env_for("follower-postgres", BLACKBOX_FOLLOWER_DB_NAME) _print_header("Running source production database migrations") c.run("bundle exec rails db:migrate", env=source_env, pty=True) _print_header("Running follower production database migrations") c.run("bundle exec rails db:migrate", env=follower_env, pty=True) _print_header(f"Waiting for source app healthcheck at {source_base_url}") _wait_for_http(base_url=source_base_url, timeout=120) _print_header(f"Waiting for follower app healthcheck at {follower_base_url}") _wait_for_http(base_url=follower_base_url, timeout=120) _print_header("Running source/follower network smoke") _blackbox_network_smoke(c, source_base_url) _blackbox_network_smoke(c, follower_base_url) _blackbox_service_to_service_smoke(c, source_env, "source-app", follower_base_url) _blackbox_service_to_service_smoke(c, follower_env, "follower-app", source_base_url) _print_header("Bootstrapping confirmed E2E users on source and follower") _bootstrap_blackbox_users(c, source_env, email, password, username, alt_email, alt_password, alt_username) _bootstrap_blackbox_users(c, follower_env, email, password, username, alt_email, alt_password, alt_username) _print_header("Running follower sync HTTP E2E against production images") _run_blackbox_rspec( source_base_url, email, password, username, alt_email, alt_password, alt_username, spec_path=[ "spec/e2e/http/tournament_follow_sync_spec.rb", "spec/e2e/http/tournament_follow_sync_websocket_spec.rb", "spec/e2e/http/tournament_follow_sync_live_websocket_spec.rb", "spec/e2e/http/tournament_timer_follow_sync_websocket_spec.rb", "spec/e2e/http/tournament_follow_sync_beamers_websocket_spec.rb", ], extra_env={ "TURNIERE_SOURCE_E2E_BASE_URL": source_base_url, "TURNIERE_FOLLOWER_E2E_BASE_URL": follower_base_url, }, ) @task(name="blackbox-production") def blackbox_production( c, host_port=SERVER_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, build=True, keep_running=False, ): """Build the production image, boot a Postgres-backed stack, run HTTP E2E, and print a summary.""" env = _compose_env( host_port=host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, expect_profiling="false", ) exit_error = None try: docker_blackbox_up( c, host_port=host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, build=build, ) docker_blackbox_test( c, host_port=host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) except (RuntimeError, subprocess.CalledProcessError) as error: exit_error = error _print_service_logs(env, "app") _print_service_logs(env, "postgres") finally: if keep_running: _print_header("Stack kept running") print( f"Production app: http://127.0.0.1:{host_port}\n" f"Stop it later with: {_format_command(_docker_compose_cmd() + ['down', '-v', '--remove-orphans'])}" ) else: docker_blackbox_down( c, host_port=host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) if exit_error: raise exit_error @task(name="docker-test-http-e2e") def docker_test_http_e2e( c, image=TEST_TAG, email=E2E_EMAIL, password=E2E_PASSWORD, username=E2E_USERNAME, ): """Run the HTTP E2E suite inside the test Docker image against a live Rails server.""" command = ( f"docker run --rm {image} bash -lc " f"\"cd /app && " "bundle exec rails db:migrate && " "bundle exec rails runner -e test " f"\\\"user = User.find_or_initialize_by(email: '{email}'); " f"user.username = '{username}'; " f"user.password = '{password}'; " f"user.password_confirmation = '{password}'; " "user.confirmed_at = Time.current; " "user.uid = user.email; " "user.provider = 'email'; " "user.save!\\\" && " f"bundle exec rails s -e test -p {SERVER_PORT} -b 0.0.0.0 >/tmp/turniere-e2e-server.log 2>&1 & " "SERVER_PID=\\$! && " f"ruby script/wait_for_http.rb {TEST_BASE_URL} 60 && " f"TURNIERE_E2E_BASE_URL={TEST_BASE_URL} " f"TURNIERE_E2E_EMAIL={email} " f"TURNIERE_E2E_PASSWORD={password} " f"TURNIERE_E2E_USERNAME={username} " "bundle exec rspec spec/e2e/http && " "kill \\$SERVER_PID\"" ) c.run(command, pty=True) @task(name="docker-test-shard") def docker_test_shard(c, node_index=1, node_total=8, image=TEST_TAG, build=True): """Run one Rails-spec shard inside the test Docker image.""" if build: docker_build_all(c) command = ( f"docker run --rm {image} bash -lc " f"\"cd /app && bundle exec rails db:migrate && " f"inv test-shard --node-index={node_index} --node-total={node_total}\"" ) c.run(command, pty=True) @task(name="manual-db-migrate") def manual_db_migrate(c): """Migrate the funky manual test database without touching db/test.sqlite3.""" setup(c) c.run( _bundle_exec("rails", "db:migrate"), env=_manual_test_env(), pty=True, ) @task(name="manual-db-seed") def manual_db_seed(c, reset=False): """Fill the funky manual test DB with reusable E2E scenario data.""" setup(c) if reset: for suffix in ("", "-shm", "-wal", ".journal"): candidate = Path(f"{MANUAL_TEST_DB_PATH}{suffix}") if candidate.exists(): candidate.unlink() manual_db_migrate(c) c.run( _bootstrap_user_command(E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME), env=_manual_test_env(), pty=True, ) c.run( _bootstrap_user_command(E2E_ALT_EMAIL, E2E_ALT_PASSWORD, E2E_ALT_USERNAME), env=_manual_test_env(), pty=True, ) process, log_handle = _start_server( f"{_bundle_exec('rails', 's')} -e test -b 127.0.0.1 -p {MANUAL_TEST_PORT}", _manual_test_env(), log_path="/tmp/turniere-manual-db-seed-server.log", ) try: _wait_for_http(base_url=MANUAL_TEST_BASE_URL, timeout=60) _seed_via_e2e_scenarios(c, MANUAL_TEST_BASE_URL, base_env=_manual_test_env()) print(f"Manual DB ready: {MANUAL_TEST_DB_PATH}") finally: _stop_process(process, log_handle) @task(name="manual-server") def manual_server(c, host=SERVER_HOST, port=MANUAL_TEST_PORT): """Run local Rails test server against the funky manual DB.""" manual_db_migrate(c) c.run( f"{_bundle_exec('rails', 's')} -e test -b {host} -p {port}", env=_manual_test_env(), pty=True, ) @task(name="docker-blackbox-follow-seed-leader") def docker_blackbox_follow_seed_leader( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, ): """Seed the running source/leader Postgres app with reusable E2E scenario data.""" env = _follow_compose_env( source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) _print_header("Seeding source/leader Postgres through HTTP E2E scenarios") for email, password, username, scenario, stop_at in _e2e_seed_scenarios(): _print_header(f"Seeding leader {scenario} as {email} until {stop_at}") command = [ "run", "--rm", "-e", f"TURNIERE_E2E_BASE_URL={BLACKBOX_SOURCE_INTERNAL_BASE_URL}", "-e", f"TURNIERE_E2E_EMAIL={email}", "-e", f"TURNIERE_E2E_PASSWORD={password}", "-e", f"TURNIERE_E2E_USERNAME={username}", "e2e", "ruby", "script/e2e_scenarios.rb", scenario, "--stop-at", stop_at, ] full_command = _docker_compose_cmd( project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, ) + command print(_format_command(full_command), flush=True) result = _stream_subprocess(full_command, env=env, check=False) if result.returncode != 0: raise subprocess.CalledProcessError( result.returncode, result.args, output=result.stdout, stderr=result.stderr, ) @task(name="docker-blackbox-follow-manual-seeded-up") def docker_blackbox_follow_manual_seeded_up( c, source_host_port=BLACKBOX_SOURCE_HOST_PORT, follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, app_image=PRODUCTION_TAG, runner_image=TEST_TAG, postgres_image=BLACKBOX_POSTGRES_IMAGE, build=True, ): """Boot source/follower production stack, migrate/bootstrap users, then seed leader Postgres.""" docker_blackbox_follow_manual_up( c, source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, build=build, ) docker_blackbox_follow_seed_leader( c, source_host_port=source_host_port, follower_host_port=follower_host_port, app_image=app_image, runner_image=runner_image, postgres_image=postgres_image, ) _print_header("Seeded leader/follower manual setup ready") print(f"Leader URL: http://127.0.0.1:{source_host_port}") print(f"Follower URL: http://127.0.0.1:{follower_host_port}") print(f"Stop it with: inv docker-blackbox-follow-down")