diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e7e51a..0f698a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -110,6 +110,9 @@ rails follow sync e2e spec: SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod MAILGUN_API_KEY: blackbox-test-api-key MAILGUN_DOMAIN: blackbox.example.com + TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: "3" + NO_PROXY: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 RAILS_LOG_TO_STDOUT: "1" RAILS_SERVE_STATIC_FILES: "1" command: @@ -130,6 +133,9 @@ rails follow sync e2e spec: SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod MAILGUN_API_KEY: blackbox-test-api-key MAILGUN_DOMAIN: blackbox.example.com + TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: "3" + NO_PROXY: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 RAILS_LOG_TO_STDOUT: "1" RAILS_SERVE_STATIC_FILES: "1" command: diff --git a/app/services/tournament_sync_pusher.rb b/app/services/tournament_sync_pusher.rb index aad9fea..bcb02b1 100644 --- a/app/services/tournament_sync_pusher.rb +++ b/app/services/tournament_sync_pusher.rb @@ -6,6 +6,7 @@ require 'uri' class TournamentSyncPusher class SyncFailed < StandardError; end + DEFAULT_HTTP_TIMEOUT = 5 def self.push!(tournament) new(tournament).push! @@ -55,7 +56,15 @@ class TournamentSyncPusher request.body = JSON.dump({ snapshot: snapshot }) Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http| + timeout = http_timeout_seconds + http.open_timeout = timeout + http.read_timeout = timeout + http.write_timeout = timeout if http.respond_to?(:write_timeout=) http.request(request) end end + + def http_timeout_seconds + ENV.fetch('TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS', DEFAULT_HTTP_TIMEOUT).to_i + end end diff --git a/docker-compose.blackbox-follow.yml b/docker-compose.blackbox-follow.yml index cee253c..debad6a 100644 --- a/docker-compose.blackbox-follow.yml +++ b/docker-compose.blackbox-follow.yml @@ -39,6 +39,7 @@ services: SECRET_KEY_BASE: ${TURNIERE_BLACKBOX_SECRET_KEY_BASE:-turniere-blackbox-secret-key-base-please-change-in-real-prod} MAILGUN_API_KEY: ${TURNIERE_BLACKBOX_MAILGUN_API_KEY:-blackbox-test-api-key} MAILGUN_DOMAIN: ${TURNIERE_BLACKBOX_MAILGUN_DOMAIN:-blackbox.example.com} + TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: ${TURNIERE_BLACKBOX_TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS:-3} RAILS_LOG_TO_STDOUT: "1" RAILS_SERVE_STATIC_FILES: "1" ports: @@ -56,6 +57,7 @@ services: SECRET_KEY_BASE: ${TURNIERE_BLACKBOX_SECRET_KEY_BASE:-turniere-blackbox-secret-key-base-please-change-in-real-prod} MAILGUN_API_KEY: ${TURNIERE_BLACKBOX_MAILGUN_API_KEY:-blackbox-test-api-key} MAILGUN_DOMAIN: ${TURNIERE_BLACKBOX_MAILGUN_DOMAIN:-blackbox.example.com} + TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: ${TURNIERE_BLACKBOX_TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS:-3} RAILS_LOG_TO_STDOUT: "1" RAILS_SERVE_STATIC_FILES: "1" ports: diff --git a/spec/e2e/http/tournament_follow_sync_spec.rb b/spec/e2e/http/tournament_follow_sync_spec.rb index ed8e5d9..d5fcc44 100644 --- a/spec/e2e/http/tournament_follow_sync_spec.rb +++ b/spec/e2e/http/tournament_follow_sync_spec.rb @@ -213,7 +213,7 @@ RSpec.describe 'Tournament follower sync HTTP E2E' do end end - def wait_for_tournament_sync!(source_tournament_id:, follower_tournament_id:, timeout: 90) + def wait_for_tournament_sync!(source_tournament_id:, follower_tournament_id:, timeout: 20) deadline = Time.now + timeout last_progress_at = Time.now @@ -222,7 +222,7 @@ RSpec.describe 'Tournament follower sync HTTP E2E' do follower = fetch_tournament(client: follower_anonymous_client, tournament_id: follower_tournament_id) return follower if tournament_signature(source) == tournament_signature(follower) - if Time.now - last_progress_at >= 5 + if Time.now - last_progress_at >= 2 puts "sync wait source=#{sync_progress_signature(source).inspect} follower=#{sync_progress_signature(follower).inspect}" $stdout.flush last_progress_at = Time.now diff --git a/tasks.py b/tasks.py index 09a8459..16a2dfc 100644 --- a/tasks.py +++ b/tasks.py @@ -239,6 +239,48 @@ def _print_header(title): print(f"\n== {title} ==", flush=True) +def _blackbox_network_smoke(c, target_base_url): + command = [ + "ruby", + "-rnet/http", + "-ruri", + "-e", + ( + "uri = URI.parse(ARGV.fetch(0) + '/healthz'); " + "res = Net::HTTP.get_response(uri); " + "abort(\"healthcheck #{res.code}\") unless res.is_a?(Net::HTTPSuccess); " + "puts(\"network smoke ok #{uri}\")" + ), + target_base_url, + ] + c.run(_format_command(command), pty=True) + + +def _blackbox_service_to_service_smoke(c, env, source_label, target_base_url): + command = [ + "bundle", + "exec", + "rails", + "runner", + ( + "require 'net/http'; " + "require 'uri'; " + "uri = URI.parse(ENV.fetch('BLACKBOX_SMOKE_TARGET_URL') + '/healthz'); " + "Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 3, read_timeout: 3) do |http| " + "res = http.get(uri.request_uri); " + "abort(\"service smoke #{res.code}\") unless res.is_a?(Net::HTTPSuccess); " + "puts(\"service smoke ok #{ENV.fetch('BLACKBOX_SMOKE_SOURCE_LABEL')} -> #{uri}\"); " + "end" + ), + ] + smoke_env = env.copy() + smoke_env.update( + BLACKBOX_SMOKE_SOURCE_LABEL=source_label, + BLACKBOX_SMOKE_TARGET_URL=target_base_url, + ) + c.run(_format_command(command), env=smoke_env, pty=True) + + def _bootstrap_user_command(email, password, username): return ( f"{_bundle_exec('rails', 'runner')} " @@ -1516,6 +1558,12 @@ def blackbox_follow_service( _print_header(f"Waiting for follower app healthcheck at {follower_base_url}") _wait_for_http(base_url=follower_base_url, timeout=120) + _print_header("Running source/follower network smoke") + _blackbox_network_smoke(c, source_base_url) + _blackbox_network_smoke(c, follower_base_url) + _blackbox_service_to_service_smoke(c, source_env, "source-app", follower_base_url) + _blackbox_service_to_service_smoke(c, follower_env, "follower-app", source_base_url) + _print_header("Bootstrapping confirmed E2E users on source and follower") _bootstrap_blackbox_users(c, source_env, email, password, username, alt_email, alt_password, alt_username) _bootstrap_blackbox_users(c, follower_env, email, password, username, alt_email, alt_password, alt_username)