Stabilize follow sync CI network checks

.gitlab-ci.yml

@@ -110,6 +110,9 @@ rails follow sync e2e spec:
         SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod
         MAILGUN_API_KEY: blackbox-test-api-key
         MAILGUN_DOMAIN: blackbox.example.com
+        TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: "3"
+        NO_PROXY: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1
+        no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1
         RAILS_LOG_TO_STDOUT: "1"
         RAILS_SERVE_STATIC_FILES: "1"
       command:
@@ -130,6 +133,9 @@ rails follow sync e2e spec:
         SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod
         MAILGUN_API_KEY: blackbox-test-api-key
         MAILGUN_DOMAIN: blackbox.example.com
+        TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: "3"
+        NO_PROXY: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1
+        no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1
         RAILS_LOG_TO_STDOUT: "1"
         RAILS_SERVE_STATIC_FILES: "1"
       command:

app/services/tournament_sync_pusher.rb

@@ -6,6 +6,7 @@ require 'uri'
 
 class TournamentSyncPusher
   class SyncFailed < StandardError; end
+  DEFAULT_HTTP_TIMEOUT = 5
 
   def self.push!(tournament)
     new(tournament).push!
@@ -55,7 +56,15 @@ class TournamentSyncPusher
     request.body = JSON.dump({ snapshot: snapshot })
 
     Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+      timeout = http_timeout_seconds
+      http.open_timeout = timeout
+      http.read_timeout = timeout
+      http.write_timeout = timeout if http.respond_to?(:write_timeout=)
       http.request(request)
     end
   end
+
+  def http_timeout_seconds
+    ENV.fetch('TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS', DEFAULT_HTTP_TIMEOUT).to_i
+  end
 end

docker-compose.blackbox-follow.yml

@@ -39,6 +39,7 @@ services:
       SECRET_KEY_BASE: ${TURNIERE_BLACKBOX_SECRET_KEY_BASE:-turniere-blackbox-secret-key-base-please-change-in-real-prod}
       MAILGUN_API_KEY: ${TURNIERE_BLACKBOX_MAILGUN_API_KEY:-blackbox-test-api-key}
       MAILGUN_DOMAIN: ${TURNIERE_BLACKBOX_MAILGUN_DOMAIN:-blackbox.example.com}
+      TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: ${TURNIERE_BLACKBOX_TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS:-3}
       RAILS_LOG_TO_STDOUT: "1"
       RAILS_SERVE_STATIC_FILES: "1"
     ports:
@@ -56,6 +57,7 @@ services:
       SECRET_KEY_BASE: ${TURNIERE_BLACKBOX_SECRET_KEY_BASE:-turniere-blackbox-secret-key-base-please-change-in-real-prod}
       MAILGUN_API_KEY: ${TURNIERE_BLACKBOX_MAILGUN_API_KEY:-blackbox-test-api-key}
       MAILGUN_DOMAIN: ${TURNIERE_BLACKBOX_MAILGUN_DOMAIN:-blackbox.example.com}
+      TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: ${TURNIERE_BLACKBOX_TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS:-3}
       RAILS_LOG_TO_STDOUT: "1"
       RAILS_SERVE_STATIC_FILES: "1"
     ports:

spec/e2e/http/tournament_follow_sync_spec.rb

@@ -213,7 +213,7 @@ RSpec.describe 'Tournament follower sync HTTP E2E' do
     end
   end
 
-  def wait_for_tournament_sync!(source_tournament_id:, follower_tournament_id:, timeout: 90)
+  def wait_for_tournament_sync!(source_tournament_id:, follower_tournament_id:, timeout: 20)
     deadline = Time.now + timeout
     last_progress_at = Time.now
 
@@ -222,7 +222,7 @@ RSpec.describe 'Tournament follower sync HTTP E2E' do
       follower = fetch_tournament(client: follower_anonymous_client, tournament_id: follower_tournament_id)
       return follower if tournament_signature(source) == tournament_signature(follower)
 
-      if Time.now - last_progress_at >= 5
+      if Time.now - last_progress_at >= 2
         puts "sync wait source=#{sync_progress_signature(source).inspect} follower=#{sync_progress_signature(follower).inspect}"
         $stdout.flush
         last_progress_at = Time.now

tasks.py

@@ -239,6 +239,48 @@ def _print_header(title):
     print(f"\n== {title} ==", flush=True)
 
 
+def _blackbox_network_smoke(c, target_base_url):
+    command = [
+        "ruby",
+        "-rnet/http",
+        "-ruri",
+        "-e",
+        (
+            "uri = URI.parse(ARGV.fetch(0) + '/healthz'); "
+            "res = Net::HTTP.get_response(uri); "
+            "abort(\"healthcheck #{res.code}\") unless res.is_a?(Net::HTTPSuccess); "
+            "puts(\"network smoke ok #{uri}\")"
+        ),
+        target_base_url,
+    ]
+    c.run(_format_command(command), pty=True)
+
+
+def _blackbox_service_to_service_smoke(c, env, source_label, target_base_url):
+    command = [
+        "bundle",
+        "exec",
+        "rails",
+        "runner",
+        (
+            "require 'net/http'; "
+            "require 'uri'; "
+            "uri = URI.parse(ENV.fetch('BLACKBOX_SMOKE_TARGET_URL') + '/healthz'); "
+            "Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 3, read_timeout: 3) do |http| "
+            "res = http.get(uri.request_uri); "
+            "abort(\"service smoke #{res.code}\") unless res.is_a?(Net::HTTPSuccess); "
+            "puts(\"service smoke ok #{ENV.fetch('BLACKBOX_SMOKE_SOURCE_LABEL')} -> #{uri}\"); "
+            "end"
+        ),
+    ]
+    smoke_env = env.copy()
+    smoke_env.update(
+        BLACKBOX_SMOKE_SOURCE_LABEL=source_label,
+        BLACKBOX_SMOKE_TARGET_URL=target_base_url,
+    )
+    c.run(_format_command(command), env=smoke_env, pty=True)
+
+
 def _bootstrap_user_command(email, password, username):
     return (
         f"{_bundle_exec('rails', 'runner')} "
@@ -1516,6 +1558,12 @@ def blackbox_follow_service(
     _print_header(f"Waiting for follower app healthcheck at {follower_base_url}")
     _wait_for_http(base_url=follower_base_url, timeout=120)
 
+    _print_header("Running source/follower network smoke")
+    _blackbox_network_smoke(c, source_base_url)
+    _blackbox_network_smoke(c, follower_base_url)
+    _blackbox_service_to_service_smoke(c, source_env, "source-app", follower_base_url)
+    _blackbox_service_to_service_smoke(c, follower_env, "follower-app", source_base_url)
+
     _print_header("Bootstrapping confirmed E2E users on source and follower")
     _bootstrap_blackbox_users(c, source_env, email, password, username, alt_email, alt_password, alt_username)
     _bootstrap_blackbox_users(c, follower_env, email, password, username, alt_email, alt_password, alt_username)