diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9112e83..e43e9cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -88,3 +88,90 @@ rails e2e spec: script: - cd /app - inv blackbox-service + +rails follow sync e2e spec: + stage: test + tags: + - docker + image: $CI_REGISTRY_IMAGE/test/commits:$CI_COMMIT_SHA + services: + - name: postgres:16 + alias: source-postgres + variables: + POSTGRES_DB: turniere_source_blackbox + POSTGRES_USER: turniere + POSTGRES_PASSWORD: turniere + - name: postgres:16 + alias: follower-postgres + variables: + POSTGRES_DB: turniere_follower_blackbox + POSTGRES_USER: turniere + POSTGRES_PASSWORD: turniere + - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA + alias: source-app + variables: + POSTGRES_URL: postgres://turniere:turniere@source-postgres:5432/turniere_source_blackbox + POSTGRES_USERNAME: turniere + POSTGRES_PASSWORD: turniere + SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod + MAILGUN_API_KEY: blackbox-test-api-key + MAILGUN_DOMAIN: blackbox.example.com + TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: "3" + NO_PROXY: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + RAILS_LOG_TO_STDOUT: "1" + RAILS_SERVE_STATIC_FILES: "1" + command: + - bundle + - exec + - rails + - s + - -p + - "3000" + - -b + - 0.0.0.0 + - name: $CI_REGISTRY_IMAGE/production/commits:$CI_COMMIT_SHA + alias: follower-app + variables: + POSTGRES_URL: postgres://turniere:turniere@follower-postgres:5432/turniere_follower_blackbox + POSTGRES_USERNAME: turniere + POSTGRES_PASSWORD: turniere + SECRET_KEY_BASE: turniere-blackbox-secret-key-base-please-change-in-real-prod + MAILGUN_API_KEY: blackbox-test-api-key + MAILGUN_DOMAIN: blackbox.example.com + TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: "3" + NO_PROXY: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + RAILS_LOG_TO_STDOUT: "1" + RAILS_SERVE_STATIC_FILES: "1" + command: + - bundle + - exec + - rails + - s + - -p + - "3000" + - -b + - 0.0.0.0 + variables: + FF_NETWORK_PER_BUILD: "true" + NO_PROXY: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + TURNIERE_SOURCE_E2E_BASE_URL: http://source-app:3000 + TURNIERE_FOLLOWER_E2E_BASE_URL: http://follower-app:3000 + TURNIERE_E2E_EMAIL: e2e@example.com + TURNIERE_E2E_PASSWORD: password123 + TURNIERE_E2E_USERNAME: e2e-user + rules: + - if: $SKIP_TEST + when: never + - if: $SKIP_ALL_SPECS + when: never + - if: $SKIP_FOLLOW_E2E_SPEC + when: never + - if: $CI_PIPELINE_SOURCE != "push" + when: never + - when: always + script: + - cd /app + - inv blackbox-follow-service diff --git a/README.md b/README.md index 68c6ad0..90bf18f 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ Running tests works as follows: bundle exec rspec ``` +Leader/follower tournament sync docs: + +- [doc/leader_follower.md](doc/leader_follower.md) + ## Docker [Registry](https://gitlab.com/turniere/turniere-backend/container_registry) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 4820771..b1bddc6 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -30,6 +30,16 @@ class ApplicationController < ActionController::API }, status: :forbidden end + def render_read_only_error + render json: { + error: 'Tournament is in read only follower mode' + }, status: :locked + end + + def require_writable_tournament!(tournament) + render_read_only_error if tournament.read_only_mode? + end + def render_not_found_error(exception) render json: { error: exception.to_s }, status: :not_found end diff --git a/app/controllers/beamers_controller.rb b/app/controllers/beamers_controller.rb index 7dba9a3..74df102 100644 --- a/app/controllers/beamers_controller.rb +++ b/app/controllers/beamers_controller.rb @@ -5,6 +5,7 @@ class BeamersController < ApplicationController before_action :set_beamer, only: %i[show update destroy] before_action :authenticate_user!, only: %i[create update destroy] before_action -> { require_owner! @tournament.owner }, only: %i[create update destroy] + before_action -> { require_writable_tournament!(@tournament) }, only: %i[create update destroy] rescue_from ActiveRecord::RecordNotFound, with: :render_not_found_error # GET /tournaments/:tournament_id/beamers @@ -32,6 +33,7 @@ class BeamersController < ApplicationController def create beamer = @tournament.beamers.new(beamer_params) if beamer.save + push_sync_if_needed! render json: beamer, status: :created else render json: beamer.errors, status: :unprocessable_entity @@ -41,6 +43,7 @@ class BeamersController < ApplicationController # PATCH /tournaments/:tournament_id/beamers/:id def update if @beamer.update(beamer_params) + push_sync_if_needed! render json: @beamer else render json: @beamer.errors, status: :unprocessable_entity @@ -50,6 +53,7 @@ class BeamersController < ApplicationController # DELETE /tournaments/:tournament_id/beamers/:id def destroy @beamer.destroy + push_sync_if_needed! head :no_content end @@ -66,4 +70,8 @@ class BeamersController < ApplicationController def beamer_params params.permit(:name, :display_state, :qr_code_url, :logo_url) end + + def push_sync_if_needed! + TournamentSyncEnqueue.call(@tournament) + end end diff --git a/app/controllers/bets_controller.rb b/app/controllers/bets_controller.rb index 3a63c3e..876bf56 100644 --- a/app/controllers/bets_controller.rb +++ b/app/controllers/bets_controller.rb @@ -3,6 +3,7 @@ class BetsController < ApplicationController before_action :set_match, only: %i[index create] before_action :authenticate_user!, only: %i[create] + before_action -> { require_writable_tournament!(@match.tournament) }, only: %i[create] rescue_from UserServiceError, with: :handle_user_service_error def index diff --git a/app/controllers/match_scores_controller.rb b/app/controllers/match_scores_controller.rb index f4d3014..d239aef 100644 --- a/app/controllers/match_scores_controller.rb +++ b/app/controllers/match_scores_controller.rb @@ -4,6 +4,7 @@ class MatchScoresController < ApplicationController before_action :set_match_score, only: %i[show update] before_action :authenticate_user!, only: %i[update] before_action -> { require_owner! @match_score.owner }, only: %i[update] + before_action -> { require_writable_tournament!(@match_score.tournament) }, only: %i[update] # GET /scores/1 def show @@ -14,6 +15,7 @@ class MatchScoresController < ApplicationController def update if @match_score.update(match_score_params) UpdateGroupsGroupScoresAndSave.call(group: @match_score.match.group) if @match_score.part_of_group_match? + push_sync_if_needed!(@match_score.tournament) render json: @match_score else render json: @match_score.errors, status: :unprocessable_entity @@ -31,4 +33,8 @@ class MatchScoresController < ApplicationController def match_score_params params.slice(:points, :hidden_points).permit! end + + def push_sync_if_needed!(tournament) + TournamentSyncEnqueue.call(tournament) + end end diff --git a/app/controllers/matches_controller.rb b/app/controllers/matches_controller.rb index 1317ead..0d605fd 100644 --- a/app/controllers/matches_controller.rb +++ b/app/controllers/matches_controller.rb @@ -7,6 +7,8 @@ class MatchesController < ApplicationController before_action -> { require_owner! @match.owner }, only: %i[update] before_action -> { require_owner! @group.owner }, only: %i[create] before_action :set_tournament, only: %i[index] + before_action -> { require_writable_tournament!(@match.tournament) }, only: %i[update] + before_action -> { require_writable_tournament!(@group.tournament) }, only: %i[create] # GET/tournaments/1/matches def index @@ -54,6 +56,7 @@ class MatchesController < ApplicationController render json: match.errors, status: :unprocessable_entity elsif match.save UpdateGroupsGroupScoresAndSave.call(group: @group) + push_sync_if_needed!(@group.tournament) render json: match, include: %w[match_scores.team group], status: :created else render json: match.errors, status: :unprocessable_entity @@ -63,6 +66,7 @@ class MatchesController < ApplicationController # PATCH/PUT /matches/1 def update new_state = match_params['state'] + synced_tournament = nil Match.transaction do if @match.update(match_params) @@ -75,11 +79,13 @@ class MatchesController < ApplicationController end render json: @match + synced_tournament = @match.tournament else render json: @match.errors, status: :unprocessable_entity raise ActiveRecord::Rollback end end + push_sync_if_needed!(synced_tournament) unless synced_tournament.nil? end private @@ -156,4 +162,8 @@ class MatchesController < ApplicationController end end end + + def push_sync_if_needed!(tournament) + TournamentSyncEnqueue.call(tournament) + end end diff --git a/app/controllers/stages_controller.rb b/app/controllers/stages_controller.rb index 800204d..fcdac4d 100644 --- a/app/controllers/stages_controller.rb +++ b/app/controllers/stages_controller.rb @@ -4,6 +4,7 @@ class StagesController < ApplicationController before_action :set_stage, only: %i[show update] before_action :authenticate_user!, only: %i[update] before_action -> { require_owner! @stage.owner }, only: %i[update] + before_action -> { require_writable_tournament!(@stage.tournament) }, only: %i[update] # GET /stages/1 def show @@ -18,16 +19,19 @@ class StagesController < ApplicationController return end + synced_tournament = nil Stage.transaction do if @stage.update(stage_params) handle_group_stage_end render json: @stage + synced_tournament = @stage.tournament else render json: @stage.errors, status: :unprocessable_entity raise ActiveRecord::Rollback end end + push_sync_if_needed!(synced_tournament) unless synced_tournament.nil? else render json: { error: 'The state attribute may only be changed to finished' @@ -75,4 +79,8 @@ class StagesController < ApplicationController def stage_params params.slice(:state).permit! end + + def push_sync_if_needed!(tournament) + TournamentSyncEnqueue.call(tournament) + end end diff --git a/app/controllers/teams_controller.rb b/app/controllers/teams_controller.rb index 522820d..7b5d7b5 100644 --- a/app/controllers/teams_controller.rb +++ b/app/controllers/teams_controller.rb @@ -4,6 +4,7 @@ class TeamsController < ApplicationController before_action :set_team, only: %i[show update] before_action :authenticate_user!, only: %i[update] before_action -> { require_owner! @team.owner }, only: %i[update] + before_action -> { require_writable_tournament!(@team.tournament) }, only: %i[update] # GET /teams/1 def show @@ -13,6 +14,7 @@ class TeamsController < ApplicationController # PATCH/PUT /teams/1 def update if @team.update(team_params) + push_sync_if_needed!(@team.tournament) render json: @team else render json: @team.errors, status: :unprocessable_entity @@ -28,4 +30,8 @@ class TeamsController < ApplicationController def team_params params.slice(:name).permit! end + + def push_sync_if_needed!(tournament) + TournamentSyncEnqueue.call(tournament) + end end diff --git a/app/controllers/tournaments_controller.rb b/app/controllers/tournaments_controller.rb index b706d9e..a95b245 100644 --- a/app/controllers/tournaments_controller.rb +++ b/app/controllers/tournaments_controller.rb @@ -12,12 +12,16 @@ class TournamentsController < ApplicationController end before_action :set_tournament_for_show, only: %i[show] - before_action :set_tournament, only: %i[update destroy set_timer_end timer_end] + before_action :set_tournament, only: %i[update destroy set_timer_end timer_end sync_state] before_action :authenticate_user!, only: %i[create update destroy set_timer_end] before_action -> { require_owner! @tournament.owner }, only: %i[update destroy set_timer_end] before_action :validate_create_params, only: %i[create] before_action :validate_update_params, only: %i[update] before_action :validate_set_timer_end_params, only: %i[set_timer_end] + before_action :ensure_update_allowed!, only: %i[update] + before_action :ensure_destroy_allowed!, only: %i[destroy] + before_action :ensure_timer_update_allowed!, only: %i[set_timer_end] + before_action :authorize_sync_request!, only: %i[sync_state] rescue_from ActiveRecord::RecordNotFound, with: :render_not_found_error # GET /tournaments @@ -52,11 +56,20 @@ class TournamentsController < ApplicationController # POST /tournaments def create params = tournament_params - params.require(:teams) group_stage = params.delete(:group_stage) teams = params.delete('teams') - # create tournament - tournament = current_user.tournaments.new params + tournament = current_user.tournaments.new params.except(:teams, :group_stage) + + if tournament.read_only_mode? + if tournament.save + render json: tournament, status: :created, location: tournament + else + render json: tournament.errors, status: :unprocessable_entity + end + return + end + + raise ActionController::ParameterMissing, :teams if teams.nil? if group_stage params.require(:playoff_teams_amount) groups = organize_teams_in_groups(teams) @@ -87,6 +100,7 @@ class TournamentsController < ApplicationController # PATCH/PUT /tournaments/1 def update + synced_tournament = nil Tournament.transaction do if only_playoff_teams_amount_changed @tournament.instant_finalists_amount, @tournament.intermediate_round_participants_amount = @@ -96,12 +110,14 @@ class TournamentsController < ApplicationController ) end if @tournament.update(tournament_params) + synced_tournament = @tournament render json: @tournament else render json: @tournament.errors, status: :unprocessable_entity raise ActiveRecord::Rollback end end + push_sync_if_needed!(synced_tournament) unless synced_tournament.nil? end # DELETE /tournaments/1 @@ -117,12 +133,20 @@ class TournamentsController < ApplicationController # PATCH /tournaments/:id/set_timer_end def set_timer_end if @tournament.update(timer_end_params) + push_sync_if_needed!(@tournament) render json: @tournament else render json: @tournament.errors, status: :unprocessable_entity end end + def sync_state + TournamentSnapshotImporter.import!(tournament: @tournament, snapshot: sync_state_params) + render json: @tournament, include: '**' + rescue KeyError, ArgumentError => e + render json: { error: e.message }, status: :unprocessable_entity + end + private @@ -194,10 +218,16 @@ class TournamentsController < ApplicationController end def tournament_params - params.slice(:name, :description, :public, :teams, :group_stage, :playoff_teams_amount).permit! + params.slice( + :name, :description, :public, :teams, :group_stage, :playoff_teams_amount, + :instant_finalists_amount, :intermediate_round_participants_amount, + :read_only_mode, :sync_target_url, :sync_auth_token + ).permit! end def validate_create_params + return if ActiveModel::Type::Boolean.new.cast(params[:read_only_mode]) + teams = params['teams'] return if teams.is_a?(Array) && teams.reject { |t| t.is_a? ActionController::Parameters }.count.zero? @@ -211,12 +241,21 @@ class TournamentsController < ApplicationController end def validate_update_params + return if sync_config_only_update? + return if only_playoff_teams_amount_changed - playoff_teams_amount = params['playoff_teams_amount'].to_i || @tournament.playoff_teams_amount - instant_finalists_amount = params['instant_finalists_amount'].to_i || @tournament.instant_finalists_amount - intermediate_round_participants_amount = params['intermediate_round_participants_amount'].to_i || - @tournament.intermediate_round_participants_amount + playoff_teams_amount = params.key?('playoff_teams_amount') ? params['playoff_teams_amount'].to_i : @tournament.playoff_teams_amount + instant_finalists_amount = if params.key?('instant_finalists_amount') + params['instant_finalists_amount'].to_i + else + @tournament.instant_finalists_amount + end + intermediate_round_participants_amount = if params.key?('intermediate_round_participants_amount') + params['intermediate_round_participants_amount'].to_i + else + @tournament.intermediate_round_participants_amount + end return if instant_finalists_amount + (intermediate_round_participants_amount / 2) == playoff_teams_amount @@ -225,6 +264,46 @@ class TournamentsController < ApplicationController error: 'playoff_teams_amount, instant_finalists_amount and intermediate_round_participants_amount don\'t match' }, status: :unprocessable_entity end + def sync_state_params + params.require(:snapshot).permit!.to_h + end + + def authorize_sync_request! + return render json: { error: 'Tournament is not configured to accept sync pushes' }, status: :unprocessable_entity unless @tournament.sync_accepts_push? + + token = request.authorization.to_s.delete_prefix('Bearer ').presence || request.headers['X-Tournament-Sync-Token'].to_s + return render json: { error: 'Missing sync token' }, status: :unauthorized if token.blank? + + matches = token.bytesize == @tournament.sync_auth_token.to_s.bytesize && + ActiveSupport::SecurityUtils.secure_compare(token, @tournament.sync_auth_token.to_s) + return if matches + + render json: { error: 'Invalid sync token' }, status: :unauthorized + end + + def ensure_update_allowed! + return unless @tournament.read_only_mode? + return if sync_config_only_update? + + render_read_only_error + end + + def ensure_destroy_allowed! + require_writable_tournament!(@tournament) + end + + def ensure_timer_update_allowed! + require_writable_tournament!(@tournament) + end + + def sync_config_only_update? + allowed = %w[id controller action format tournament read_only_mode sync_target_url sync_auth_token] + (params.keys - allowed).empty? + end + + def push_sync_if_needed!(tournament) + TournamentSyncEnqueue.call(tournament) + end def validate_set_timer_end_params timer_end = params[:timer_end] diff --git a/app/models/group.rb b/app/models/group.rb index b9bac6a..547a12b 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -6,6 +6,7 @@ class Group < ApplicationRecord has_many :group_scores, dependent: :destroy delegate :owner, to: :stage + delegate :tournament, to: :stage def teams matches.map(&:teams).flatten.uniq diff --git a/app/models/match.rb b/app/models/match.rb index e80697a..21d0a6f 100644 --- a/app/models/match.rb +++ b/app/models/match.rb @@ -22,6 +22,10 @@ class Match < ApplicationRecord stage ? stage.owner : group.owner end + def tournament + stage ? stage.tournament : group.stage.tournament + end + def current_leading_team return nil if match_scores.size < 2 return nil if score_value_for(match_scores.first) == score_value_for(match_scores.second) diff --git a/app/models/match_score.rb b/app/models/match_score.rb index 5bf815a..5d4cbc1 100644 --- a/app/models/match_score.rb +++ b/app/models/match_score.rb @@ -5,6 +5,7 @@ class MatchScore < ApplicationRecord belongs_to :team delegate :owner, to: :match + delegate :tournament, to: :match def part_of_group_match? match.group_match? diff --git a/app/models/tournament.rb b/app/models/tournament.rb index af67d0c..4b284d8 100644 --- a/app/models/tournament.rb +++ b/app/models/tournament.rb @@ -10,6 +10,8 @@ class Tournament < ApplicationRecord validates :name, presence: true validates :code, presence: true, uniqueness: true + validates :sync_target_url, format: URI::DEFAULT_PARSER.make_regexp(%w[http https]), allow_blank: true + validate :sync_configuration_is_complete validate :playoff_teams_amount_is_positive_power_of_two @@ -17,6 +19,7 @@ class Tournament < ApplicationRecord after_initialize :generate_code after_create_commit :ensure_default_beamer! + before_validation :clear_follower_sync_token_when_disabling_read_only_mode def matches [stages.map(&:matches), stages.map { |s| s.groups.map(&:matches) }].flatten @@ -34,6 +37,14 @@ class Tournament < ApplicationRecord stages.find { |stage| stage.level == -1 } end + def sync_push_enabled? + sync_target_url.present? && sync_auth_token.present? && !read_only_mode? + end + + def sync_accepts_push? + read_only_mode? && sync_auth_token.present? + end + private def generate_code @@ -52,9 +63,32 @@ class Tournament < ApplicationRecord 'playoff_teams_amount needs to be a positive power of two') end + def sync_configuration_is_complete + return if sync_target_url.blank? && sync_auth_token.blank? + return if sync_target_url.present? && sync_auth_token.present? + return if read_only_mode? && sync_auth_token.present? && sync_target_url.blank? + + errors.add(:base, 'sync_target_url and sync_auth_token must be configured together') + end + def ensure_default_beamer! beamers.find_or_create_by!(name: Beamer::DEFAULT_NAME) do |beamer| beamer.display_state = 'matches_all' end end + + def sync_configuration_is_complete + return if sync_target_url.blank? && sync_auth_token.blank? + return if sync_target_url.present? && sync_auth_token.present? + return if read_only_mode? && sync_auth_token.present? && sync_target_url.blank? + + errors.add(:base, 'sync_target_url and sync_auth_token must be configured together') + end + + def clear_follower_sync_token_when_disabling_read_only_mode + return if read_only_mode? + return if sync_target_url.present? + + self.sync_auth_token = nil + end end diff --git a/app/models/tournament_sync_queue_entry.rb b/app/models/tournament_sync_queue_entry.rb new file mode 100644 index 0000000..297cd4b --- /dev/null +++ b/app/models/tournament_sync_queue_entry.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +class TournamentSyncQueueEntry < ApplicationRecord + LOCK_TTL = 2.minutes + MAX_BACKOFF = 5.minutes + + belongs_to :tournament + + validates :snapshot_json, presence: true + validates :status, presence: true + + scope :due, -> { where(status: 'pending').where('next_attempt_at <= ?', Time.current) } + scope :unlocked, lambda { + where(locked_at: nil).or(where('locked_at < ?', Time.current - LOCK_TTL)) + } + + def snapshot + JSON.parse(snapshot_json, symbolize_names: true) + end + + def schedule_retry!(error_message) + update!( + status: 'pending', + last_error: error_message, + attempts: attempts + 1, + last_attempt_at: Time.current, + next_attempt_at: Time.current + retry_delay, + locked_at: nil, + lock_token: nil + ) + end + + def mark_synced! + update!( + status: 'synced', + last_error: nil, + last_attempt_at: Time.current, + next_attempt_at: Time.current, + attempts: 0, + locked_at: nil, + lock_token: nil + ) + end + + def acquire_lock! + token = SecureRandom.hex(8) + updated = self.class + .where(id: id) + .where(locked_at: nil) + .or(self.class.where(id: id).where('locked_at < ?', Time.current - LOCK_TTL)) + .update_all(locked_at: Time.current, lock_token: token) + return nil if updated.zero? + + reload + token + end + + private + + def retry_delay + [2**attempts, MAX_BACKOFF].min.seconds + end +end diff --git a/app/services/tournament_snapshot_builder.rb b/app/services/tournament_snapshot_builder.rb new file mode 100644 index 0000000..a1b4ee4 --- /dev/null +++ b/app/services/tournament_snapshot_builder.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +class TournamentSnapshotBuilder + ROOT_ATTRIBUTES = TournamentSyncSchema.synced_columns_for(Tournament).map(&:to_sym).freeze + TEAM_ATTRIBUTES = TournamentSyncSchema.synced_columns_for(Team).map(&:to_sym).freeze + BEAMER_ATTRIBUTES = TournamentSyncSchema.synced_columns_for(Beamer).map(&:to_sym).freeze + STAGE_ATTRIBUTES = TournamentSyncSchema.synced_columns_for(Stage).map(&:to_sym).freeze + GROUP_ATTRIBUTES = TournamentSyncSchema.synced_columns_for(Group).map(&:to_sym).freeze + MATCH_ATTRIBUTES = TournamentSyncSchema.synced_columns_for(Match).map(&:to_sym).freeze + MATCH_SCORE_ATTRIBUTES = TournamentSyncSchema.synced_columns_for(MatchScore).map(&:to_sym).freeze + GROUP_SCORE_ATTRIBUTES = TournamentSyncSchema.synced_columns_for(GroupScore).map(&:to_sym).freeze + + def self.build(tournament) + new(tournament).build + end + + def initialize(tournament) + @tournament = tournament + end + + def build + payload = tournament.attributes.symbolize_keys.slice(*ROOT_ATTRIBUTES) + payload[:timer_end] = tournament.timer_end&.iso8601 + payload[:sync_generated_at] = Time.current.iso8601(6) + payload[:teams] = teams_payload + payload[:beamers] = beamers_payload + payload[:stages] = stages_payload + payload[:groups] = groups_payload + payload[:matches] = matches_payload + payload[:match_scores] = match_scores_payload + payload[:group_scores] = group_scores_payload + payload + end + + private + + attr_reader :tournament + + def teams_payload + teams_for_snapshot.sort_by(&:id).map do |team| + team.attributes.symbolize_keys.slice(*TEAM_ATTRIBUTES) + end + end + + def stages_payload + tournament.stages.order(:id).map do |stage| + stage.attributes.symbolize_keys.slice(*STAGE_ATTRIBUTES) + end + end + + def beamers_payload + tournament.beamers.order(:id).map do |beamer| + beamer.attributes.symbolize_keys.slice(*BEAMER_ATTRIBUTES) + end + end + + def groups_payload + tournament.stages.flat_map(&:groups).sort_by(&:id).map do |group| + group.attributes.symbolize_keys.slice(*GROUP_ATTRIBUTES) + end + end + + def matches_payload + tournament.matches.sort_by(&:id).map do |match| + match.attributes.symbolize_keys.slice(*MATCH_ATTRIBUTES) + end + end + + def match_scores_payload + tournament.matches.flat_map(&:match_scores).sort_by(&:id).map do |match_score| + match_score.attributes.symbolize_keys.slice(*MATCH_SCORE_ATTRIBUTES) + end + end + + def group_scores_payload + tournament.stages.flat_map(&:groups).flat_map(&:group_scores).sort_by(&:id).map do |group_score| + group_score.attributes.symbolize_keys.slice(*GROUP_SCORE_ATTRIBUTES) + end + end + + def teams_for_snapshot + ( + tournament.teams.to_a + + tournament.matches.flat_map(&:teams) + + tournament.stages.flat_map(&:groups).flat_map(&:group_scores).map(&:team) + ).uniq(&:id) + end +end diff --git a/app/services/tournament_snapshot_importer.rb b/app/services/tournament_snapshot_importer.rb new file mode 100644 index 0000000..72630d7 --- /dev/null +++ b/app/services/tournament_snapshot_importer.rb @@ -0,0 +1,205 @@ +# frozen_string_literal: true + +class TournamentSnapshotImporter + ROOT_ATTRIBUTES = %i[ + name + code + description + public + playoff_teams_amount + instant_finalists_amount + intermediate_round_participants_amount + ].freeze + + def self.import!(tournament:, snapshot:) + new(tournament: tournament, snapshot: snapshot).import! + end + + def initialize(tournament:, snapshot:) + @tournament = tournament + @snapshot = snapshot.deep_symbolize_keys + end + + def import! + Tournament.transaction do + tournament.lock! + return tournament if stale_snapshot? + + existing_teams = tournament.teams.to_a + existing_beamers = tournament.beamers.to_a + existing_stages = tournament.stages.to_a + existing_groups = existing_stages.flat_map(&:groups) + existing_matches = tournament.matches + existing_match_scores = existing_matches.flat_map(&:match_scores) + existing_group_scores = existing_groups.flat_map(&:group_scores) + + update_tournament! + teams_by_source_id = sync_teams! + sync_beamers! + stages_by_source_id = sync_stages! + groups_by_source_id = sync_groups!(stages_by_source_id) + matches_by_source_id = sync_matches!(stages_by_source_id, groups_by_source_id) + sync_match_scores!(matches_by_source_id, teams_by_source_id) + sync_group_scores!(groups_by_source_id, teams_by_source_id) + + destroy_stale_records(existing_beamers, snapshot.fetch(:beamers, [])) + destroy_stale_records(existing_group_scores, snapshot.fetch(:group_scores, [])) + destroy_stale_records(existing_match_scores, snapshot.fetch(:match_scores, [])) + destroy_stale_records(existing_matches, snapshot.fetch(:matches, [])) + destroy_stale_records(existing_groups, snapshot.fetch(:groups, [])) + destroy_stale_records(existing_stages, snapshot.fetch(:stages, [])) + destroy_stale_records(existing_teams, snapshot.fetch(:teams, [])) + end + + tournament + end + + private + + attr_reader :tournament, :snapshot + + def update_tournament! + root_attrs = snapshot.slice(*ROOT_ATTRIBUTES) + root_attrs[:code] = tournament.code if code_taken_by_another_tournament?(root_attrs[:code]) + root_attrs[:timer_end] = parse_time(snapshot[:timer_end]) + root_attrs[:sync_source_tournament_id] = snapshot[:id] + root_attrs[:sync_last_imported_snapshot_at] = snapshot_generated_at + tournament.update!(root_attrs) + end + + def sync_teams! + sync_records( + model_class: Team, + existing_records: tournament.teams.to_a, + payload: snapshot.fetch(:teams, []) + ) do |team, attrs| + team.tournament = tournament + team.name = attrs.fetch(:name) + end + end + + def sync_stages! + sync_records( + model_class: Stage, + existing_records: tournament.stages.to_a, + payload: snapshot.fetch(:stages, []) + ) do |stage, attrs| + stage.tournament = tournament + stage.level = attrs.fetch(:level) + stage.state = attrs.fetch(:state) + end + end + + def sync_beamers! + existing_by_source_id = tournament.beamers.index_by(&:sync_source_id) + existing_by_name = tournament.beamers.index_by(&:name) + + snapshot.fetch(:beamers, []).each_with_object({}) do |attrs, beamers_by_source_id| + source_id = attrs.fetch(:id) + beamer = existing_by_source_id[source_id] || existing_by_name[attrs.fetch(:name)] || Beamer.new + beamer.sync_source_id = source_id + beamer.tournament = tournament + beamer.name = attrs.fetch(:name) + beamer.display_state = attrs.fetch(:display_state) + beamer.is_default = attrs.fetch(:is_default, false) + beamer.qr_code_url = attrs[:qr_code_url] + beamer.logo_url = attrs[:logo_url] + beamer.save! + beamers_by_source_id[source_id] = beamer + end + end + + def sync_groups!(stages_by_source_id) + sync_records( + model_class: Group, + existing_records: tournament.stages.flat_map(&:groups), + payload: snapshot.fetch(:groups, []) + ) do |group, attrs| + group.stage = stages_by_source_id.fetch(attrs.fetch(:stage_id)) + group.number = attrs.fetch(:number) + end + end + + def sync_matches!(stages_by_source_id, groups_by_source_id) + sync_records( + model_class: Match, + existing_records: tournament.matches, + payload: snapshot.fetch(:matches, []) + ) do |match, attrs| + match.stage = attrs[:stage_id] ? stages_by_source_id.fetch(attrs[:stage_id]) : nil + match.group = attrs[:group_id] ? groups_by_source_id.fetch(attrs[:group_id]) : nil + match.state = attrs.fetch(:state) + match.position = attrs[:position] + match.decider = attrs[:decider] + end + end + + def sync_match_scores!(matches_by_source_id, teams_by_source_id) + sync_records( + model_class: MatchScore, + existing_records: tournament.matches.flat_map(&:match_scores), + payload: snapshot.fetch(:match_scores, []) + ) do |match_score, attrs| + match_score.match = matches_by_source_id.fetch(attrs.fetch(:match_id)) + match_score.team = teams_by_source_id.fetch(attrs.fetch(:team_id)) + match_score.points = attrs.fetch(:points) + match_score.hidden_points = attrs.fetch(:hidden_points, 0) + end + end + + def sync_group_scores!(groups_by_source_id, teams_by_source_id) + sync_records( + model_class: GroupScore, + existing_records: tournament.stages.flat_map(&:groups).flat_map(&:group_scores), + payload: snapshot.fetch(:group_scores, []) + ) do |group_score, attrs| + group_score.group = groups_by_source_id.fetch(attrs.fetch(:group_id)) + group_score.team = teams_by_source_id.fetch(attrs.fetch(:team_id)) + group_score.group_points = attrs.fetch(:group_points) + group_score.scored_points = attrs.fetch(:scored_points) + group_score.received_points = attrs.fetch(:received_points) + group_score.position = attrs.fetch(:position, 0) + end + end + + def sync_records(model_class:, existing_records:, payload:) + existing_by_source_id = existing_records.index_by(&:sync_source_id) + + payload.each_with_object({}) do |attrs, records_by_source_id| + source_id = attrs.fetch(:id) + record = existing_by_source_id[source_id] || model_class.new + record.sync_source_id = source_id + yield(record, attrs) + record.save! + records_by_source_id[source_id] = record + end + end + + def destroy_stale_records(existing_records, payload) + source_ids = payload.map { |attrs| attrs.fetch(:id) } + stale_records = existing_records.reject { |record| source_ids.include?(record.sync_source_id) } + stale_records.each(&:destroy!) + end + def parse_time(value) + return nil if value.blank? + + Time.zone.parse(value) + end + + def snapshot_generated_at + parse_time(snapshot[:sync_generated_at]) + end + + def stale_snapshot? + return false if snapshot_generated_at.nil? + return false if tournament.sync_last_imported_snapshot_at.nil? + + snapshot_generated_at <= tournament.sync_last_imported_snapshot_at + end + + def code_taken_by_another_tournament?(code) + return false if code.blank? + + Tournament.where(code: code).where.not(id: tournament.id).exists? + end +end diff --git a/app/services/tournament_sync_enqueue.rb b/app/services/tournament_sync_enqueue.rb new file mode 100644 index 0000000..f2694b7 --- /dev/null +++ b/app/services/tournament_sync_enqueue.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class TournamentSyncEnqueue + def self.call(tournament) + new(tournament).call + end + + def initialize(tournament) + @tournament = tournament + end + + def call + return false unless tournament.sync_push_enabled? + + payload = TournamentSnapshotBuilder.build(fresh_tournament) + entry = TournamentSyncQueueEntry.find_or_initialize_by(tournament: tournament) + entry.snapshot_json = JSON.dump(payload) + entry.status = 'pending' + entry.attempts = 0 + entry.last_attempt_at = nil + entry.next_attempt_at = Time.current + entry.last_error = nil + entry.locked_at = nil + entry.lock_token = nil + entry.save! + TournamentSyncWorker.start + TournamentSyncWorker.wake + true + end + + private + + attr_reader :tournament + + def fresh_tournament + Tournament.find(tournament.id) + end +end diff --git a/app/services/tournament_sync_processor.rb b/app/services/tournament_sync_processor.rb new file mode 100644 index 0000000..605f952 --- /dev/null +++ b/app/services/tournament_sync_processor.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class TournamentSyncProcessor + def self.process_due! + new.process_due! + end + + def process_due! + return unless queue_table_available? + + loop do + entry = next_entry + break if entry.nil? + + process_entry(entry) + end + end + + private + + def next_entry + TournamentSyncQueueEntry.due.unlocked.order(:next_attempt_at, :id).first + end + + def process_entry(entry) + lock_token = entry.acquire_lock! + return if lock_token.nil? + + entry.reload + TournamentSyncPusher.push_snapshot!(entry.tournament, entry.snapshot) + entry.mark_synced! + rescue TournamentSyncPusher::SyncFailed => e + entry.reload.schedule_retry!(e.message) + end + + def queue_table_available? + TournamentSyncQueueEntry.connection.data_source_exists?(TournamentSyncQueueEntry.table_name) + rescue ActiveRecord::ActiveRecordError, PG::Error + false + end +end diff --git a/app/services/tournament_sync_pusher.rb b/app/services/tournament_sync_pusher.rb new file mode 100644 index 0000000..bcb02b1 --- /dev/null +++ b/app/services/tournament_sync_pusher.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'uri' + +class TournamentSyncPusher + class SyncFailed < StandardError; end + DEFAULT_HTTP_TIMEOUT = 5 + + def self.push!(tournament) + new(tournament).push! + end + + def self.push_snapshot!(tournament, snapshot) + new(tournament).push_snapshot!(snapshot) + end + + def initialize(tournament) + @tournament = tournament + end + + def push! + return false unless tournament.sync_push_enabled? + + push_snapshot!(TournamentSnapshotBuilder.build(tournament)) + end + + def push_snapshot!(snapshot) + response = perform_request(snapshot: snapshot) + + unless response.is_a?(Net::HTTPSuccess) + raise SyncFailed, "sync push failed with status #{response.code}: #{response.body}" + end + + tournament.update_columns(sync_last_pushed_at: Time.current, sync_last_push_error: nil) + true + rescue SyncFailed => e + tournament.update_columns(sync_last_push_error: e.message) + raise + rescue StandardError => e + tournament.update_columns(sync_last_push_error: e.message) + raise SyncFailed, e.message + end + + private + + attr_reader :tournament + + def perform_request(snapshot:) + uri = URI.parse(tournament.sync_target_url) + request = Net::HTTP::Patch.new(uri) + request['accept'] = 'application/json' + request['content-type'] = 'application/json' + request['authorization'] = "Bearer #{tournament.sync_auth_token}" + request.body = JSON.dump({ snapshot: snapshot }) + + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http| + timeout = http_timeout_seconds + http.open_timeout = timeout + http.read_timeout = timeout + http.write_timeout = timeout if http.respond_to?(:write_timeout=) + http.request(request) + end + end + + def http_timeout_seconds + ENV.fetch('TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS', DEFAULT_HTTP_TIMEOUT).to_i + end +end diff --git a/app/services/tournament_sync_schema.rb b/app/services/tournament_sync_schema.rb new file mode 100644 index 0000000..230c908 --- /dev/null +++ b/app/services/tournament_sync_schema.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class TournamentSyncSchema + MODEL_COLUMNS = { + Tournament => { + synced: %w[ + id + name + code + description + public + playoff_teams_amount + instant_finalists_amount + intermediate_round_participants_amount + timer_end + ].freeze, + ignored: %w[ + user_id + created_at + updated_at + read_only_mode + sync_target_url + sync_auth_token + sync_source_tournament_id + sync_last_pushed_at + sync_last_push_error + sync_last_imported_snapshot_at + ].freeze + }.freeze, + Team => { + synced: %w[id name].freeze, + ignored: %w[tournament_id created_at updated_at sync_source_id].freeze + }.freeze, + Beamer => { + synced: %w[id name display_state is_default qr_code_url logo_url].freeze, + ignored: %w[tournament_id created_at updated_at sync_source_id].freeze + }.freeze, + Stage => { + synced: %w[id level state].freeze, + ignored: %w[tournament_id created_at updated_at sync_source_id].freeze + }.freeze, + Group => { + synced: %w[id number stage_id].freeze, + ignored: %w[created_at updated_at sync_source_id].freeze + }.freeze, + Match => { + synced: %w[id state position stage_id group_id decider].freeze, + ignored: %w[created_at updated_at sync_source_id].freeze + }.freeze, + MatchScore => { + synced: %w[id points match_id team_id hidden_points].freeze, + ignored: %w[created_at updated_at sync_source_id].freeze + }.freeze, + GroupScore => { + synced: %w[id group_points scored_points received_points team_id group_id position].freeze, + ignored: %w[created_at updated_at sync_source_id].freeze + }.freeze + }.freeze + + class << self + def synced_columns_for(model_class) + MODEL_COLUMNS.fetch(model_class).fetch(:synced) + end + + def ignored_columns_for(model_class) + MODEL_COLUMNS.fetch(model_class).fetch(:ignored) + end + end +end diff --git a/app/services/tournament_sync_worker.rb b/app/services/tournament_sync_worker.rb new file mode 100644 index 0000000..87b733d --- /dev/null +++ b/app/services/tournament_sync_worker.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +class TournamentSyncWorker + POLL_INTERVAL = 2.seconds + + class << self + def start + return if Rails.env.test? + + mutex.synchronize do + return if running? + + @thread = Thread.new do + Thread.current.name = 'tournament-sync-worker' if Thread.current.respond_to?(:name=) + loop do + TournamentSyncProcessor.process_due! + wait_for_wake_or_timeout + rescue StandardError => e + Rails.logger.warn("Tournament sync worker error: #{e.message}") + wait_for_wake_or_timeout + end + end + end + end + + def wake + mutex.synchronize do + condition.broadcast + end + end + + def running? + @thread&.alive? + end + + private + + def mutex + @mutex ||= Mutex.new + end + + def condition + @condition ||= ConditionVariable.new + end + + def wait_for_wake_or_timeout + mutex.synchronize do + condition.wait(mutex, POLL_INTERVAL) + end + end + end +end diff --git a/config/initializers/tournament_sync_worker.rb b/config/initializers/tournament_sync_worker.rb new file mode 100644 index 0000000..b0407a9 --- /dev/null +++ b/config/initializers/tournament_sync_worker.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +Rails.application.config.after_initialize do + TournamentSyncWorker.start unless Rails.env.test? +end diff --git a/config/routes.rb b/config/routes.rb index 18aa961..f4170ac 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -25,6 +25,7 @@ Rails.application.routes.draw do member do get :timer_end patch :set_timer_end + patch :sync_state end end resources :match_scores, only: %i[show update] diff --git a/db/migrate/20260413100000_add_tournament_sync_support.rb b/db/migrate/20260413100000_add_tournament_sync_support.rb new file mode 100644 index 0000000..b421dda --- /dev/null +++ b/db/migrate/20260413100000_add_tournament_sync_support.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class AddTournamentSyncSupport < ActiveRecord::Migration[7.0] + def change + change_table :tournaments, bulk: true do |t| + t.boolean :read_only_mode, null: false, default: false + t.string :sync_target_url + t.string :sync_auth_token + t.integer :sync_source_tournament_id + t.datetime :sync_last_pushed_at + t.string :sync_last_push_error + end + + change_table :teams, bulk: true do |t| + t.integer :sync_source_id + end + + change_table :stages, bulk: true do |t| + t.integer :sync_source_id + end + + change_table :groups, bulk: true do |t| + t.integer :sync_source_id + end + + change_table :matches, bulk: true do |t| + t.integer :sync_source_id + end + + change_table :match_scores, bulk: true do |t| + t.integer :sync_source_id + end + + change_table :group_scores, bulk: true do |t| + t.integer :sync_source_id + end + end +end diff --git a/db/migrate/20260413210000_create_tournament_sync_queue_entries.rb b/db/migrate/20260413210000_create_tournament_sync_queue_entries.rb new file mode 100644 index 0000000..b06c3f9 --- /dev/null +++ b/db/migrate/20260413210000_create_tournament_sync_queue_entries.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class CreateTournamentSyncQueueEntries < ActiveRecord::Migration[7.0] + def change + create_table :tournament_sync_queue_entries do |t| + t.references :tournament, null: false, foreign_key: { on_delete: :cascade }, index: { unique: true } + t.text :snapshot_json, null: false + t.datetime :next_attempt_at, null: false + t.datetime :last_attempt_at + t.datetime :locked_at + t.string :lock_token + t.integer :attempts, null: false, default: 0 + t.string :status, null: false, default: 'pending' + t.string :last_error + + t.timestamps + end + + add_index :tournament_sync_queue_entries, :next_attempt_at + add_index :tournament_sync_queue_entries, :status + end +end diff --git a/db/migrate/20260414103000_add_sync_snapshot_tracking_to_tournaments.rb b/db/migrate/20260414103000_add_sync_snapshot_tracking_to_tournaments.rb new file mode 100644 index 0000000..d0d453b --- /dev/null +++ b/db/migrate/20260414103000_add_sync_snapshot_tracking_to_tournaments.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddSyncSnapshotTrackingToTournaments < ActiveRecord::Migration[7.0] + def change + add_column :tournaments, :sync_last_imported_snapshot_at, :datetime + end +end diff --git a/db/migrate/20260416131500_add_sync_source_id_to_beamers.rb b/db/migrate/20260416131500_add_sync_source_id_to_beamers.rb new file mode 100644 index 0000000..8ef90e3 --- /dev/null +++ b/db/migrate/20260416131500_add_sync_source_id_to_beamers.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddSyncSourceIdToBeamers < ActiveRecord::Migration[7.0] + def change + add_column :beamers, :sync_source_id, :integer + end +end diff --git a/db/migrate/20260416143000_ensure_tournament_sync_columns_exist.rb b/db/migrate/20260416143000_ensure_tournament_sync_columns_exist.rb new file mode 100644 index 0000000..0e70b6e --- /dev/null +++ b/db/migrate/20260416143000_ensure_tournament_sync_columns_exist.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class EnsureTournamentSyncColumnsExist < ActiveRecord::Migration[7.0] + def change + add_column_if_missing :tournaments, :read_only_mode, :boolean, null: false, default: false + add_column_if_missing :tournaments, :sync_target_url, :string + add_column_if_missing :tournaments, :sync_auth_token, :string + add_column_if_missing :tournaments, :sync_source_tournament_id, :integer + add_column_if_missing :tournaments, :sync_last_pushed_at, :datetime + add_column_if_missing :tournaments, :sync_last_push_error, :string + + add_column_if_missing :teams, :sync_source_id, :integer + add_column_if_missing :stages, :sync_source_id, :integer + add_column_if_missing :groups, :sync_source_id, :integer + add_column_if_missing :matches, :sync_source_id, :integer + add_column_if_missing :match_scores, :sync_source_id, :integer + add_column_if_missing :group_scores, :sync_source_id, :integer + end + + private + + def add_column_if_missing(table, column, type, **options) + return if column_exists?(table, column) + + add_column(table, column, type, **options) + end +end diff --git a/db/migrate/20260416144000_ensure_tournament_sync_queue_entries_exist.rb b/db/migrate/20260416144000_ensure_tournament_sync_queue_entries_exist.rb new file mode 100644 index 0000000..fa67a5e --- /dev/null +++ b/db/migrate/20260416144000_ensure_tournament_sync_queue_entries_exist.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class EnsureTournamentSyncQueueEntriesExist < ActiveRecord::Migration[7.0] + def change + return if table_exists?(:tournament_sync_queue_entries) + + create_table :tournament_sync_queue_entries do |t| + t.integer :tournament_id, null: false + t.text :snapshot_json, null: false + t.datetime :next_attempt_at, null: false + t.datetime :last_attempt_at + t.datetime :locked_at + t.string :lock_token + t.integer :attempts, default: 0, null: false + t.string :status, default: 'pending', null: false + t.string :last_error + t.timestamps null: false + end + + add_index :tournament_sync_queue_entries, :next_attempt_at + add_index :tournament_sync_queue_entries, :status + add_index :tournament_sync_queue_entries, :tournament_id, unique: true + add_foreign_key :tournament_sync_queue_entries, :tournaments, on_delete: :cascade + end +end diff --git a/doc/leader_follower.md b/doc/leader_follower.md new file mode 100644 index 0000000..632e805 --- /dev/null +++ b/doc/leader_follower.md @@ -0,0 +1,228 @@ +# Leader/Follower Tournament Sync + +Backend supports one-way tournament replication: + +- leader tournament stays writable +- follower tournament stays read only +- leader pushes full snapshots asynchronously +- follower never pulls +- follower never needs to reach leader + +This fits live topology: + +- leader = local laptop / Raspberry Pi / LAN host without public IP +- follower = internet VM with public HTTPS endpoint +- only leader needs outbound access to follower + +## Live Setup + +### 1. Create follower tournament on remote backend + +Create empty tournament with: + +- `read_only_mode: true` +- `sync_auth_token: ` + +Do not send `teams` payload for follower. Backend allows empty read only follower tournament creation. + +### 2. Create normal leader tournament on local backend + +Create tournament as usual. This remains normal writable tournament. + +### 3. Configure leader to push to follower + +Update leader tournament with: + +- `sync_target_url: https://remote.example/tournaments/:id/sync_state` +- `sync_auth_token: ` + +Leader becomes push source. Follower accepts only authenticated snapshot pushes. + +### 4. Run tournament normally on leader + +These leader-side actions enqueue async snapshot push: + +- tournament updates +- timer updates +- team renames +- match state changes +- match score changes +- stage finish / playoff generation + +Queue retries with backoff until follower accepts snapshot. + +## Local Manual Testing + +Backend now has local manual setup target: + +```bash +inv docker-blackbox-follow-manual-up +``` + +This starts: + +- leader production image on `http://127.0.0.1:3002` +- follower production image on `http://127.0.0.1:3003` +- separate leader and follower Postgres containers +- bootstrapped owner users on both backends + +Teardown: + +```bash +inv docker-blackbox-follow-down +``` + +### Local topology + +Compose setup now separates DB traffic: + +- `source-postgres` only on `source-db` internal network +- `follower-postgres` only on `follower-db` internal network +- both apps share only `sync` network + +This gives: + +- separate production app containers +- separate production DB containers +- DB isolation between leader and follower + +Important limit: + +- Docker Compose does **not** enforce true one-way HTTP reachability here +- local setup still allows follower container to reach leader container over shared `sync` network +- real production trust model still comes from backend design: + - leader pushes + - follower never pulls + - follower has no code path that contacts leader + +So local setup is good for frontend/manual testing of behavior, not strict network-policy simulation. + +### Manual frontend testing flow + +1. Run `inv docker-blackbox-follow-manual-up` +2. Open leader backend at `http://127.0.0.1:3002` +3. Open follower backend at `http://127.0.0.1:3003` +4. Log in with: + - email: `e2e@example.com` + - password: `password123` +5. On follower backend, create empty read-only follower tournament +6. On leader backend, create normal tournament +7. On leader tournament, set: + - `sync_target_url` to follower `PATCH /tournaments/:id/sync_state` + - `sync_auth_token` to same shared token used on follower +8. Point local frontend leader instance to `http://127.0.0.1:3002` +9. Point local frontend follower/remote instance to `http://127.0.0.1:3003` +10. Progress tournament on leader, verify follower mirrors state and stays read only + +Frontend team can use this setup to build: + +- owner sync configuration UI +- follower read-only UX +- status display for `sync_last_pushed_at` +- status display for `sync_last_push_error` +- takeover flow by disabling `read_only_mode` + +## Takeover + +If remote must become writable after sync phase: + +1. disable `read_only_mode` on follower +2. backend clears follower token when no longer needed +3. follower becomes normal standalone tournament again + +After takeover, leader should no longer push to old follower URL. + +## API Contract + +### Leader-side writable config + +Tournament push mode active when: + +- `sync_target_url` present +- `sync_auth_token` present +- `read_only_mode` false + +### Follower-side push receiver + +Tournament accepts push when: + +- `read_only_mode` true +- `sync_auth_token` present + +Push endpoint: + +- `PATCH /tournaments/:id/sync_state` + +Auth: + +- `Authorization: Bearer ` + +Payload: + +- `{ snapshot: ... }` + +Snapshot import replaces tournament graph using `sync_source_id` mapping, not local DB ids. + +## Frontend Integration Notes + +Frontend does not need separate "follower UI mode" for normal viewers. Remote tournament should look same for non-owner users. + +Owner-facing frontend can expose: + +- `read_only_mode` toggle +- `sync_target_url` input +- `sync_auth_token` input +- sync health display: + - `sync_last_pushed_at` + - `sync_last_push_error` + +Recommended owner UX: + +1. create remote follower first +2. copy remote `sync_auth_token` +3. paste remote `sync_state` URL into leader +4. save leader sync config +5. show last push time / error state +6. allow explicit follower takeover by disabling `read_only_mode` + +Frontend should block owner edits on follower when `read_only_mode` true. Backend already enforces this with `423 Locked`. + +## Future Feature Rule + +Any new persisted tournament state must be handled in three places: + +1. leader write path +2. snapshot export/import path +3. follower/read-only behavior + +Guardrail now exists: + +- `TournamentSyncSchema` lists every synced and ignored model column +- spec fails when new DB column appears without explicit sync decision + +This does not auto-implement sync for new features. It makes missing sync work fail loudly in tests instead of silently drifting. + +For every new tournament feature, add: + +1. schema decision in `TournamentSyncSchema` +2. snapshot builder/importer support if state must replicate +3. one follower lifecycle test or roundtrip spec that proves behavior survives sync + +## Coverage Today + +Current backend coverage includes: + +- async push queue and retry behavior +- follower auth and read-only lock +- full follower lifecycle HTTP E2E with unrelated tournaments and mismatched ids +- team renames +- multiple decider matches +- playoff generation blocked before deciders finish +- playoff generation after deciders finish +- follower takeover +- large roundtrip graph coverage for 32 groups, 4 teams each, 64-team playoffs + +Still not magic: + +- if new feature adds new state and nobody updates sync contract, tests will fail +- if new feature changes behavior but not schema, add behavior test too diff --git a/docker-compose.blackbox-follow.yml b/docker-compose.blackbox-follow.yml new file mode 100644 index 0000000..525d8c6 --- /dev/null +++ b/docker-compose.blackbox-follow.yml @@ -0,0 +1,107 @@ +services: + source-postgres: + image: ${TURNIERE_BLACKBOX_POSTGRES_IMAGE:-postgres:16} + environment: + POSTGRES_DB: ${TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB:-turniere_source_blackbox} + POSTGRES_USER: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere} + POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - turniere-blackbox-follow-source-postgres:/var/lib/postgresql/data + networks: + - source-db + + follower-postgres: + image: ${TURNIERE_BLACKBOX_POSTGRES_IMAGE:-postgres:16} + environment: + POSTGRES_DB: ${TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB:-turniere_follower_blackbox} + POSTGRES_USER: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere} + POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - turniere-blackbox-follow-follower-postgres:/var/lib/postgresql/data + networks: + - follower-db + + source-app: + image: ${TURNIERE_BLACKBOX_APP_IMAGE:-registry.gitlab.com/turniere/turniere-backend/production/commits:local} + depends_on: + source-postgres: + condition: service_healthy + environment: + POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@source-postgres:5432/${TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB:-turniere_source_blackbox} + POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere} + POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere} + SECRET_KEY_BASE: ${TURNIERE_BLACKBOX_SECRET_KEY_BASE:-turniere-blackbox-secret-key-base-please-change-in-real-prod} + MAILGUN_API_KEY: ${TURNIERE_BLACKBOX_MAILGUN_API_KEY:-blackbox-test-api-key} + MAILGUN_DOMAIN: ${TURNIERE_BLACKBOX_MAILGUN_DOMAIN:-blackbox.example.com} + TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: ${TURNIERE_BLACKBOX_TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS:-3} + RAILS_LOG_TO_STDOUT: "1" + RAILS_SERVE_STATIC_FILES: "1" + ports: + - "${TURNIERE_BLACKBOX_SOURCE_HOST_PORT:-3002}:3000" + networks: + - source-db + - sync + + follower-app: + image: ${TURNIERE_BLACKBOX_APP_IMAGE:-registry.gitlab.com/turniere/turniere-backend/production/commits:local} + depends_on: + follower-postgres: + condition: service_healthy + environment: + POSTGRES_URL: postgres://${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere}:${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere}@follower-postgres:5432/${TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB:-turniere_follower_blackbox} + POSTGRES_USERNAME: ${TURNIERE_BLACKBOX_POSTGRES_USER:-turniere} + POSTGRES_PASSWORD: ${TURNIERE_BLACKBOX_POSTGRES_PASSWORD:-turniere} + SECRET_KEY_BASE: ${TURNIERE_BLACKBOX_SECRET_KEY_BASE:-turniere-blackbox-secret-key-base-please-change-in-real-prod} + MAILGUN_API_KEY: ${TURNIERE_BLACKBOX_MAILGUN_API_KEY:-blackbox-test-api-key} + MAILGUN_DOMAIN: ${TURNIERE_BLACKBOX_MAILGUN_DOMAIN:-blackbox.example.com} + TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS: ${TURNIERE_BLACKBOX_TOURNAMENT_SYNC_HTTP_TIMEOUT_SECONDS:-3} + RAILS_LOG_TO_STDOUT: "1" + RAILS_SERVE_STATIC_FILES: "1" + ports: + - "${TURNIERE_BLACKBOX_FOLLOWER_HOST_PORT:-3003}:3000" + networks: + - follower-db + - sync + + e2e: + image: ${TURNIERE_BLACKBOX_RUNNER_IMAGE:-registry.gitlab.com/turniere/turniere-backend/test/commits:local} + depends_on: + source-app: + condition: service_started + follower-app: + condition: service_started + working_dir: /app + environment: + NO_PROXY: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + no_proxy: source-app,follower-app,source-postgres,follower-postgres,localhost,127.0.0.1 + TURNIERE_SOURCE_E2E_BASE_URL: ${TURNIERE_SOURCE_E2E_BASE_URL:-http://source-app:3000} + TURNIERE_FOLLOWER_E2E_BASE_URL: ${TURNIERE_FOLLOWER_E2E_BASE_URL:-http://follower-app:3000} + TURNIERE_E2E_EMAIL: ${TURNIERE_E2E_EMAIL:-e2e@example.com} + TURNIERE_E2E_PASSWORD: ${TURNIERE_E2E_PASSWORD:-password123} + TURNIERE_E2E_USERNAME: ${TURNIERE_E2E_USERNAME:-e2e-user} + TURNIERE_E2E_ALT_EMAIL: ${TURNIERE_E2E_ALT_EMAIL:-e2e-alt@example.com} + TURNIERE_E2E_ALT_PASSWORD: ${TURNIERE_E2E_ALT_PASSWORD:-password123} + TURNIERE_E2E_ALT_USERNAME: ${TURNIERE_E2E_ALT_USERNAME:-e2e-alt-user} + networks: + - sync + +volumes: + turniere-blackbox-follow-source-postgres: + turniere-blackbox-follow-follower-postgres: + +networks: + source-db: + internal: true + follower-db: + internal: true + sync: diff --git a/e2e/lib/api_client.rb b/e2e/lib/api_client.rb index d1a8584..0fef340 100644 --- a/e2e/lib/api_client.rb +++ b/e2e/lib/api_client.rb @@ -21,6 +21,7 @@ module TurniereE2E def initialize(base_url:) @base_url = base_url.sub(%r{/\z}, '') @auth_headers = {} + @login_credentials = nil end def register!(email:, password:, username:) @@ -39,6 +40,8 @@ module TurniereE2E end def login!(email:, password:) + @login_credentials = { email: email, password: password } + @auth_headers = {} response = request( :post, '/users/sign_in', @@ -73,7 +76,9 @@ module TurniereE2E private - def request(method, path, body: nil, params: nil, headers: {}) + def request(method, path, body: nil, params: nil, headers: {}, retry_on_unauthorized: true, refresh_session: true) + login!(**@login_credentials) if refresh_session && should_refresh_session?(method, path) + uri = URI.parse("#{base_url}#{path}") uri.query = URI.encode_www_form(params) if params && !params.empty? @@ -95,11 +100,17 @@ module TurniereE2E end parsed_json = parse_json_body(response.body) - { + result = { status: response.code.to_i, json: parsed_json, headers: response.each_header.to_h } + store_auth_headers!(result) + if retry_on_unauthorized && result[:status] == 401 && can_reauthenticate?(path) + login!(**@login_credentials) + return request(method, path, body: body, params: params, headers: headers, retry_on_unauthorized: false, refresh_session: false) + end + result end def merged_headers(headers) @@ -118,11 +129,20 @@ module TurniereE2E end def store_auth_headers!(response) - %w[access-token client uid].each_with_object({}) do |key, auth_headers| - auth_headers[key] = response[:headers][key] if response[:headers].key?(key) + %w[access-token client uid expiry token-type authorization].each_with_object({}) do |key, auth_headers| + value = response[:headers][key] + auth_headers[key] = value if response[:headers].key?(key) && !value.to_s.empty? end.tap do |headers| - @auth_headers = headers unless headers.empty? + @auth_headers = @auth_headers.merge(headers) unless headers.empty? end end + + def can_reauthenticate?(path) + !@login_credentials.nil? && path != '/users/sign_in' + end + + def should_refresh_session?(method, path) + !@login_credentials.nil? && method != :get && path != '/users/sign_in' + end end end diff --git a/spec/controllers/beamers_controller_spec.rb b/spec/controllers/beamers_controller_spec.rb index 1cdbe92..bc76847 100644 --- a/spec/controllers/beamers_controller_spec.rb +++ b/spec/controllers/beamers_controller_spec.rb @@ -164,6 +164,21 @@ RSpec.describe BeamersController, type: :controller do expect(response).to have_http_status(:forbidden) end end + + context 'on read only follower tournament' do + before(:each) do + @tournament.update!(read_only_mode: true, sync_auth_token: 'shared-secret') + apply_authentication_headers_for @owner + end + + it 'returns locked' do + patch :update, params: { + tournament_id: @tournament.to_param, id: @beamer.to_param, + display_state: 'groups' + } + expect(response).to have_http_status(:locked) + end + end end describe 'DELETE #destroy' do diff --git a/spec/controllers/stages_controller_spec.rb b/spec/controllers/stages_controller_spec.rb index 0b30611..355cd6e 100644 --- a/spec/controllers/stages_controller_spec.rb +++ b/spec/controllers/stages_controller_spec.rb @@ -84,11 +84,11 @@ RSpec.describe StagesController, type: :controller do end it 'adds new stages to the tournament' do - expect(running_group_stage.tournament.stages.size).to be > 1 + expect(running_group_stage.tournament.reload.stages.size).to be > 1 end it 'adds the right teams' do - expect(running_group_stage.tournament.stages.max_by(&:level).teams) + expect(running_group_stage.tournament.reload.stages.max_by(&:level).teams) .to match_array(GroupStageService.get_advancing_teams(running_group_stage)) end end diff --git a/spec/controllers/tournaments_controller_spec.rb b/spec/controllers/tournaments_controller_spec.rb index 7e51682..24d2fc2 100644 --- a/spec/controllers/tournaments_controller_spec.rb +++ b/spec/controllers/tournaments_controller_spec.rb @@ -224,6 +224,25 @@ RSpec.describe TournamentsController, type: :controller do apply_authentication_headers_for @user end + context 'with read_only_mode=true' do + it 'creates an empty follower tournament without teams payload' do + expect do + post :create, params: { + name: 'Follower Tournament', + public: false, + read_only_mode: true, + sync_auth_token: 'shared-secret' + } + end.to change(Tournament, :count).by(1) + + expect(response).to have_http_status(:created) + tournament = Tournament.find(deserialize_response(response)[:id]) + expect(tournament.read_only_mode?).to eq(true) + expect(tournament.teams).to be_empty + expect(tournament.stages).to be_empty + end + end + context 'with existing teams' do it 'creates a new Tournament' do expect do @@ -417,6 +436,48 @@ RSpec.describe TournamentsController, type: :controller do expect(response.media_type).to eq('application/json') end + it 'allows sync config changes on read only tournaments' do + @tournament.update!(read_only_mode: true, sync_auth_token: 'old-token') + + put :update, params: { + id: @tournament.to_param, + read_only_mode: false, + sync_target_url: 'https://remote.example.com/tournaments/1/sync_state', + sync_auth_token: 'new-token' + } + + expect(response).to have_http_status(:ok) + @tournament.reload + expect(@tournament.read_only_mode?).to eq(false) + expect(@tournament.sync_target_url).to include('/sync_state') + expect(@tournament.sync_auth_token).to eq('new-token') + end + + it 'allows follower takeover when only read_only_mode is disabled' do + @tournament.update!(read_only_mode: true, sync_auth_token: 'old-token') + request.headers['CONTENT_TYPE'] = 'application/json' + + put :update, params: { + id: @tournament.to_param, + read_only_mode: false + }, as: :json + + expect(response).to have_http_status(:ok) + @tournament.reload + expect(@tournament.read_only_mode?).to eq(false) + expect(@tournament.sync_target_url).to be_blank + expect(@tournament.sync_auth_token).to be_blank + end + + it 'blocks normal updates while tournament is read only' do + @tournament.update!(read_only_mode: true, sync_auth_token: 'shared-secret') + + put :update, params: { id: @tournament.to_param, name: 'Should Not Work' } + + expect(response).to have_http_status(:locked) + expect(deserialize_response(response)[:error]).to eq('Tournament is in read only follower mode') + end + context 'any variable relevant for group stage to playoff transition changed' do before(:each) do @filled_tournament = create(:group_stage_tournament) @@ -550,6 +611,14 @@ RSpec.describe TournamentsController, type: :controller do delete :destroy, params: { id: @tournament.to_param } end.to change(Team, :count).by(-@tournament.teams.size) end + + it 'blocks destroy while tournament is read only' do + @tournament.update!(read_only_mode: true, sync_auth_token: 'shared-secret') + + delete :destroy, params: { id: @tournament.to_param } + + expect(response).to have_http_status(:locked) + end end context 'as another user' do @@ -563,4 +632,53 @@ RSpec.describe TournamentsController, type: :controller do end end end + + describe 'PATCH #sync_state' do + before do + @tournament.update!(read_only_mode: true, sync_auth_token: 'shared-secret') + end + + it 'imports a pushed snapshot' do + request.headers['Authorization'] = 'Bearer shared-secret' + + patch :sync_state, params: { + id: @tournament.to_param, + snapshot: { + id: 123, + name: 'Synced Tournament', + code: 'abc123', + description: 'Synced description', + public: true, + playoff_teams_amount: 4, + instant_finalists_amount: 4, + intermediate_round_participants_amount: 0, + timer_end: nil, + teams: [{ id: 7, name: 'Alpha' }], + stages: [], + groups: [], + matches: [], + match_scores: [], + group_scores: [] + } + } + + expect(response).to have_http_status(:ok) + @tournament.reload + expect(@tournament.sync_source_tournament_id).to eq(123) + expect(@tournament.name).to eq('Synced Tournament') + expect(@tournament.teams.pluck(:name)).to eq(['Alpha']) + end + + it 'rejects invalid tokens' do + request.headers['Authorization'] = 'Bearer wrong-token' + + patch :sync_state, params: { + id: @tournament.to_param, + snapshot: { id: 123 } + } + + expect(response).to have_http_status(:unauthorized) + expect(deserialize_response(response)[:error]).to eq('Invalid sync token') + end + end end diff --git a/spec/e2e/http/tournament_follow_sync_spec.rb b/spec/e2e/http/tournament_follow_sync_spec.rb new file mode 100644 index 0000000..1c2efa7 --- /dev/null +++ b/spec/e2e/http/tournament_follow_sync_spec.rb @@ -0,0 +1,435 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'securerandom' +require_relative '../../../e2e/lib/api_client' + +RSpec.describe 'Tournament follower sync HTTP E2E' do + required_env_vars = %w[ + TURNIERE_E2E_EMAIL + TURNIERE_E2E_PASSWORD + TURNIERE_SOURCE_E2E_BASE_URL + TURNIERE_FOLLOWER_E2E_BASE_URL + ].freeze + + before do + missing_vars = required_env_vars.reject { |name| ENV.key?(name) } + skip("HTTP E2E requires #{missing_vars.join(', ')}") unless missing_vars.empty? + end + + let(:source_base_url) { ENV.fetch('TURNIERE_SOURCE_E2E_BASE_URL') } + let(:follower_base_url) { ENV.fetch('TURNIERE_FOLLOWER_E2E_BASE_URL') } + let(:owner_email) { ENV.fetch('TURNIERE_E2E_EMAIL') } + let(:owner_password) { ENV.fetch('TURNIERE_E2E_PASSWORD') } + let(:sync_token) { "sync-#{SecureRandom.hex(8)}" } + let(:source_owner_client) { login_client(base_url: source_base_url, email: owner_email, password: owner_password) } + let(:follower_owner_client) { login_client(base_url: follower_base_url, email: owner_email, password: owner_password) } + let(:source_anonymous_client) { TurniereE2E::ApiClient.new(base_url: source_base_url) } + let(:follower_anonymous_client) { TurniereE2E::ApiClient.new(base_url: follower_base_url) } + + it 'pushes full source lifecycle into read only follower tournament' do + log_phase!('seed unrelated tournaments') + source_noise = seed_unrelated_tournaments!(client: source_owner_client, prefix: 'Source Noise', range: 2..4) + follower_noise = seed_unrelated_tournaments!(client: follower_owner_client, prefix: 'Follower Noise', range: 5..7) + + log_phase!('create source and follower tournaments') + source = create_group_stage_tournament(client: source_owner_client, public: false, name_prefix: 'Sync Source') + follower = create_follower_tournament(client: follower_owner_client, public: true, name_prefix: 'Sync Follower') + expect(source.fetch(:id)).not_to eq(follower.fetch(:id)) + + log_phase!('configure sync and verify initial replication') + configure_sync = source_owner_client.patch("/tournaments/#{source.fetch(:id)}", body: { + sync_target_url: "#{follower_base_url}/tournaments/#{follower.fetch(:id)}/sync_state", + sync_auth_token: sync_token + }) + expect(configure_sync[:status]).to eq(200) + exercise_unrelated_requests!(source_noise: source_noise, follower_noise: follower_noise) + + source = fetch_tournament(client: source_anonymous_client, tournament_id: source.fetch(:id)) + follower = wait_for_tournament_sync!(source_tournament_id: source.fetch(:id), follower_tournament_id: follower.fetch(:id)) + expect(tournament_signature(follower)).to eq(tournament_signature(source)) + expect(synced_team_id_pairs(source: source, follower: follower)).not_to be_empty + + log_phase!('tighten playoff cutoff and rename teams') + update_cutoff = source_owner_client.patch("/tournaments/#{source.fetch(:id)}", body: { + playoff_teams_amount: 4, + instant_finalists_amount: 4, + intermediate_round_participants_amount: 0 + }) + expect(update_cutoff[:status]).to eq(200) + exercise_unrelated_requests!(source_noise: source_noise, follower_noise: follower_noise) + + source_group_stage = fetch_tournament(client: source_anonymous_client, tournament_id: source.fetch(:id)) + .fetch(:stages).find { |stage| stage.fetch(:level) == -1 } + rename_source_tournament_teams!( + source_tournament_id: source.fetch(:id), + team_names: [ + group_team_map(source_group_stage.fetch(:groups).first).fetch(1).fetch(:name), + group_team_map(source_group_stage.fetch(:groups).last).fetch(2).fetch(:name) + ] + ) + source_noise.concat(seed_unrelated_tournaments!(client: source_owner_client, prefix: 'Source Midway Noise', range: 2..3)) + follower_noise.concat(seed_unrelated_tournaments!(client: follower_owner_client, prefix: 'Follower Midway Noise', range: 3..4)) + exercise_unrelated_requests!(source_noise: source_noise, follower_noise: follower_noise) + + log_phase!('play group stage with deciders') + play_group_with_decider_lifecycle!(source_id: source.fetch(:id), groups: source_group_stage.fetch(:groups).sort_by { |group| group.fetch(:number) }) + exercise_unrelated_requests!(source_noise: source_noise, follower_noise: follower_noise) + + log_phase!('verify playoff generation replicated') + source = fetch_tournament(client: source_anonymous_client, tournament_id: source.fetch(:id)) + follower = wait_for_tournament_sync!(source_tournament_id: source.fetch(:id), follower_tournament_id: follower.fetch(:id)) + expect(tournament_signature(follower)).to eq(tournament_signature(source)) + expect(follower.fetch(:stages).map { |stage| stage.fetch(:level) }).to include(-1, 0, 1) + expect(synced_team_id_pairs(source: source, follower: follower)).not_to be_empty + + follower_group = follower.fetch(:stages).find { |stage| stage.fetch(:level) == -1 }.fetch(:groups).first + follower_team = follower_group.fetch(:matches).first.fetch(:match_scores).first.fetch(:team) + locked_team_update = follower_owner_client.patch("/teams/#{follower_team.fetch(:id)}", body: { name: 'Nope' }) + expect(locked_team_update[:status]).to eq(423) + expect(locked_team_update.dig(:json, :error)).to eq('Tournament is in read only follower mode') + + source_noise.concat(seed_unrelated_tournaments!(client: source_owner_client, prefix: 'Source Playoff Noise', range: 2..4)) + follower_noise.concat(seed_unrelated_tournaments!(client: follower_owner_client, prefix: 'Follower Playoff Noise', range: 2..5)) + exercise_unrelated_requests!(source_noise: source_noise, follower_noise: follower_noise) + + log_phase!('finish playoff bracket') + finish_playoff_bracket!(source_id: source.fetch(:id)) + exercise_unrelated_requests!(source_noise: source_noise, follower_noise: follower_noise) + + log_phase!('verify final follower state and takeover') + source = fetch_tournament(client: source_anonymous_client, tournament_id: source.fetch(:id)) + follower = wait_for_tournament_sync!(source_tournament_id: source.fetch(:id), follower_tournament_id: follower.fetch(:id)) + expect(tournament_signature(follower)).to eq(tournament_signature(source)) + expect(synced_team_id_pairs(source: source, follower: follower)).not_to be_empty + + disable_follower = follower_owner_client.patch("/tournaments/#{follower.fetch(:id)}", body: { read_only_mode: false }) + expect(disable_follower[:status]).to eq(200) + + follower_after_disable = fetch_tournament(client: follower_anonymous_client, tournament_id: follower.fetch(:id)) + writable_team = follower_after_disable.fetch(:teams).first + takeover_update = follower_owner_client.patch("/teams/#{writable_team.fetch(:id)}", body: { name: "#{writable_team.fetch(:name)} takeover" }) + expect(takeover_update[:status]).to eq(200) + end + + def log_phase!(message) + puts "phase #{message}" + $stdout.flush + end + + def login_client(base_url:, email:, password:) + client = TurniereE2E::ApiClient.new(base_url: base_url) + response = client.login!(email: email, password: password) + + expect(response[:status]).to eq(200) + expect(client.authenticated?).to eq(true) + client + end + + def create_group_stage_tournament(client:, public:, name_prefix:) + payload = { + name: "#{name_prefix} #{unique_suffix}", + description: 'Follower sync source tournament', + public: public, + group_stage: true, + playoff_teams_amount: 8, + teams: 4.times.flat_map do |group_index| + 4.times.map do |seed_index| + { + name: "#{name_prefix.tr(' ', '')}-G#{group_index + 1}-S#{seed_index + 1}", + group: group_index + } + end + end + } + + response = client.post('/tournaments', body: payload) + expect(response[:status]).to eq(201) + fetch_tournament(client: client, tournament_id: response.dig(:json, :id)) + end + + def create_playoff_tournament(client:, public:, name_prefix:) + payload = { + name: "#{name_prefix} #{unique_suffix}", + description: 'Unrelated playoff tournament', + public: public, + teams: 4.times.map do |index| + { name: "#{name_prefix.tr(' ', '')}-S#{index + 1}" } + end + } + + response = client.post('/tournaments', body: payload) + expect(response[:status]).to eq(201) + fetch_tournament(client: client, tournament_id: response.dig(:json, :id)) + end + + def create_follower_tournament(client:, public:, name_prefix:) + response = client.post('/tournaments', body: { + name: "#{name_prefix} #{unique_suffix}", + description: 'Follower tournament', + public: public, + read_only_mode: true, + sync_auth_token: sync_token + }) + expect(response[:status]).to eq(201) + fetch_tournament(client: client, tournament_id: response.dig(:json, :id)) + end + + def fetch_tournament(client:, tournament_id:) + response = client.get("/tournaments/#{tournament_id}") + expect(response[:status]).to eq(200) + response.fetch(:json) + end + + def seed_unrelated_tournaments!(client:, prefix:, range:) + amount = rand(range) + Array.new(amount) do |index| + if index.even? + create_playoff_tournament(client: client, public: false, name_prefix: "#{prefix} Playoff #{index}") + else + create_group_stage_tournament(client: client, public: true, name_prefix: "#{prefix} Groups #{index}") + end + end + end + + def exercise_unrelated_requests!(source_noise:, follower_noise:) + mutate_unrelated_tournament!(client: source_owner_client, tournaments: source_noise, suffix: 'source') + mutate_unrelated_tournament!(client: follower_owner_client, tournaments: follower_noise, suffix: 'follower') + end + + def mutate_unrelated_tournament!(client:, tournaments:, suffix:) + tournament = tournaments.sample + updated_name = "#{tournament.fetch(:name)} #{suffix} #{unique_suffix}" + update = client.patch("/tournaments/#{tournament.fetch(:id)}", body: { name: updated_name }) + expect(update[:status]).to eq(200) + + refreshed = fetch_tournament(client: client, tournament_id: tournament.fetch(:id)) + team = refreshed.fetch(:teams).first + return if team.nil? + + team_update = client.patch("/teams/#{team.fetch(:id)}", body: { name: "#{team.fetch(:name)} #{suffix}" }) + expect(team_update[:status]).to eq(200) + end + + def rename_source_tournament_teams!(source_tournament_id:, team_names:) + tournament = fetch_tournament(client: source_anonymous_client, tournament_id: source_tournament_id) + + team_names.each_with_index do |team_name, index| + team = tournament.fetch(:teams).find { |candidate| candidate.fetch(:name) == team_name } + expect(team).not_to be_nil + + response = source_owner_client.patch("/teams/#{team.fetch(:id)}", body: { + name: "#{team.fetch(:name)} Renamed #{index + 1}" + }) + expect(response[:status]).to eq(200) + tournament = fetch_tournament(client: source_anonymous_client, tournament_id: source_tournament_id) + end + end + + def wait_for_tournament_sync!(source_tournament_id:, follower_tournament_id:, timeout: 20) + deadline = Time.now + timeout + last_progress_at = Time.now + + loop do + source = fetch_tournament(client: source_anonymous_client, tournament_id: source_tournament_id) + follower = fetch_tournament(client: follower_anonymous_client, tournament_id: follower_tournament_id) + return follower if tournament_signature(source) == tournament_signature(follower) + + if Time.now - last_progress_at >= 2 + puts "sync wait source=#{sync_progress_signature(source).inspect} follower=#{sync_progress_signature(follower).inspect}" + $stdout.flush + last_progress_at = Time.now + end + + if Time.now >= deadline + raise <<~ERROR + Follower did not catch up within #{timeout}s + source_signature=#{tournament_signature(source).inspect} + follower_signature=#{tournament_signature(follower).inspect} + ERROR + end + + sleep 0.5 + end + end + + def sync_progress_signature(tournament) + { + name: tournament.fetch(:name), + stages: tournament.fetch(:stages).map { |stage| [stage.fetch(:level), stage.fetch(:state)] } + } + end + + def synced_team_id_pairs(source:, follower:) + source_by_name = source.fetch(:teams).each_with_object({}) do |team, teams_by_name| + teams_by_name[team.fetch(:name)] = team + end + follower.fetch(:teams).filter_map do |team| + source_team = source_by_name[team.fetch(:name)] + next if source_team.nil? + next if source_team.fetch(:id) == team.fetch(:id) + + [source_team.fetch(:id), team.fetch(:id)] + end + end + + def play_group_with_decider_lifecycle!(source_id:, groups:) + groups.each_with_index do |group, index| + if index < 2 + play_group_matches_for_decider!(group) + else + finish_group_clear_winner!(group) + end + end + + group_stage = fetch_tournament(client: source_anonymous_client, tournament_id: source_id) + .fetch(:stages).find { |stage| stage.fetch(:level) == -1 } + + stage_finish = source_owner_client.patch("/stages/#{group_stage.fetch(:id)}", body: { state: 'finished' }) + expect(stage_finish[:status]).to eq(422) + expect(stage_finish.dig(:json, :error)).to eq('Playoffs cannot be generated yet because a decider match is required for an advancing place') + expect(stage_finish.dig(:json, :blocking_ties).size).to be >= 2 + + stage_finish.fetch(:json).fetch(:blocking_ties).each_with_index do |blocking_tie, index| + decider_create = source_owner_client.post("/groups/#{blocking_tie.fetch(:group_id)}/matches", body: { + team_ids: blocking_tie.fetch(:team_ids) + }) + expect(decider_create[:status]).to eq(201) + decider_match = decider_create.fetch(:json) + + start_decider = source_owner_client.patch("/matches/#{decider_match.fetch(:id)}", body: { state: 'in_progress' }) + expect(start_decider[:status]).to eq(200) + decider_match.fetch(:match_scores).sort_by { |score| score.fetch(:id) }.each_with_index do |score, score_index| + response = source_owner_client.patch("/match_scores/#{score.fetch(:id)}", body: { + hidden_points: score_index.zero? ? 5 + index : 3 + }) + expect(response[:status]).to eq(200) + end + finish_decider = source_owner_client.patch("/matches/#{decider_match.fetch(:id)}", body: { state: 'finished' }) + expect(finish_decider[:status]).to eq(200) + end + + finish_stage_after_decider = source_owner_client.patch("/stages/#{group_stage.fetch(:id)}", body: { state: 'finished' }) + expect(finish_stage_after_decider[:status]).to eq(200) + end + + def play_group_matches_for_decider!(group) + teams = group_team_map(group) + play_group_match!(group, teams.fetch(1), teams.fetch(2), 1, 1) + play_group_match!(group, teams.fetch(1), teams.fetch(3), 2, 0) + play_group_match!(group, teams.fetch(1), teams.fetch(4), 2, 0) + play_group_match!(group, teams.fetch(2), teams.fetch(3), 2, 0) + play_group_match!(group, teams.fetch(2), teams.fetch(4), 2, 0) + play_group_match!(group, teams.fetch(3), teams.fetch(4), 1, 0) + end + + def finish_group_clear_winner!(group) + group.fetch(:matches).each_with_index do |match, index| + play_match!(match_id: match.fetch(:id), first_score: 10 + index, second_score: index) + end + end + + def group_team_map(group) + group.fetch(:matches) + .flat_map { |match| match.fetch(:match_scores).map { |score| score.fetch(:team) } } + .uniq { |team| team.fetch(:id) } + .sort_by { |team| team.fetch(:name) } + .each_with_index + .to_h { |team, index| [index + 1, team] } + end + + def play_group_match!(group, team_a, team_b, score_a, score_b) + match = group.fetch(:matches).find do |candidate| + team_ids = candidate.fetch(:match_scores).map { |score| score.dig(:team, :id) }.sort + team_ids == [team_a.fetch(:id), team_b.fetch(:id)].sort + end + + play_match!(match_id: match.fetch(:id), first_score: score_a, second_score: score_b) + end + + def finish_playoff_bracket!(source_id:) + loop do + tournament = fetch_tournament(client: source_anonymous_client, tournament_id: source_id) + playoff_matches = tournament.fetch(:stages) + .select { |stage| stage.fetch(:level) >= 0 } + .flat_map { |stage| stage.fetch(:matches) } + .select { |match| %w[not_started in_progress].include?(match.fetch(:state)) && match.fetch(:match_scores).size == 2 } + .sort_by { |match| [match.dig(:stage, :id) || 0, match.fetch(:position)] } + break if playoff_matches.empty? + + playoff_matches.each_with_index do |match, index| + play_match!(match_id: match.fetch(:id), first_score: 20 + index, second_score: index) + end + end + end + + def play_match!(match_id:, first_score:, second_score:) + match = source_anonymous_client.get("/matches/#{match_id}") + expect(match[:status]).to eq(200) + + unless match.dig(:json, :state) == 'in_progress' + start_response = source_owner_client.patch("/matches/#{match_id}", body: { state: 'in_progress' }) + expect(start_response[:status]).to eq(200) + match = source_anonymous_client.get("/matches/#{match_id}") + end + + ordered_scores = match.fetch(:json).fetch(:match_scores).sort_by { |score| score.fetch(:id) } + [first_score, second_score].each_with_index do |points, index| + response = source_owner_client.patch("/match_scores/#{ordered_scores[index].fetch(:id)}", body: { points: points }) + expect(response[:status]).to eq(200) + end + + finish_response = source_owner_client.patch("/matches/#{match_id}", body: { state: 'finished' }) + expect(finish_response[:status]).to eq(200) + end + + def tournament_signature(tournament) + { + name: tournament.fetch(:name), + public: tournament.fetch(:public), + playoff_teams_amount: tournament.fetch(:playoff_teams_amount), + instant_finalists_amount: tournament.fetch(:instant_finalists_amount), + intermediate_round_participants_amount: tournament.fetch(:intermediate_round_participants_amount), + timer_end: tournament[:timer_end], + teams: tournament.fetch(:teams).map { |team| team.slice(:name, :advancing_from_group_stage) }.sort_by { |team| team.fetch(:name) }, + stages: tournament.fetch(:stages).sort_by { |stage| stage.fetch(:level) }.map do |stage| + { + level: stage.fetch(:level), + state: stage.fetch(:state), + groups: stage.fetch(:groups).sort_by { |group| group.fetch(:number) }.map do |group| + { + number: group.fetch(:number), + matches: group.fetch(:matches).sort_by { |match| match.fetch(:position) }.map do |match| + match_signature(match) + end + } + end, + matches: stage.fetch(:matches).sort_by { |match| match.fetch(:position) }.map do |match| + match_signature(match) + end + } + end + } + end + + def match_signature(match) + { + state: match.fetch(:state), + position: match.fetch(:position), + decider: match.fetch(:decider), + scores: match.fetch(:match_scores).map do |score| + { + team_name: score.dig(:team, :name), + points: score.fetch(:points), + hidden_points: score.fetch(:hidden_points) + } + end.sort_by { |score| score.fetch(:team_name).to_s } + } + end + + def unique_suffix + @unique_suffix_counter ||= 0 + @unique_suffix_counter += 1 + "#{SecureRandom.hex(3)}-#{@unique_suffix_counter}" + end +end diff --git a/spec/routing/tournaments_routing_spec.rb b/spec/routing/tournaments_routing_spec.rb index f8d1acf..00bda66 100644 --- a/spec/routing/tournaments_routing_spec.rb +++ b/spec/routing/tournaments_routing_spec.rb @@ -27,5 +27,9 @@ RSpec.describe TournamentsController, type: :routing do it 'routes to #destroy' do expect(delete: '/tournaments/1').to route_to('tournaments#destroy', id: '1') end + + it 'routes to #sync_state' do + expect(patch: '/tournaments/1/sync_state').to route_to('tournaments#sync_state', id: '1') + end end end diff --git a/spec/services/tournament_snapshot_importer_spec.rb b/spec/services/tournament_snapshot_importer_spec.rb new file mode 100644 index 0000000..4788941 --- /dev/null +++ b/spec/services/tournament_snapshot_importer_spec.rb @@ -0,0 +1,225 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TournamentSnapshotImporter do + describe '.import!' do + it 'replaces follower tournament graph using source ids as stable mapping keys' do + source = create(:group_stage_tournament, group_count: 2, match_factory: :filled_group_match) + source.beamers.create!(name: 'stream-a', display_state: 'during_game', qr_code_url: 'https://example.com/qr') + follower = create(:tournament, read_only_mode: true, sync_auth_token: 'shared-secret') + + snapshot = TournamentSnapshotBuilder.build(source) + + described_class.import!(tournament: follower, snapshot: snapshot) + + follower.reload + expect(follower.sync_source_tournament_id).to eq(source.id) + expect(follower.name).to eq(source.name) + expect(follower.teams.count).to eq(snapshot.fetch(:teams).count) + expect(follower.beamers.count).to eq(snapshot.fetch(:beamers).count) + expect(follower.stages.count).to eq(source.stages.count) + expect(follower.matches.count).to eq(source.matches.count) + expect(follower.teams.pluck(:sync_source_id)).to match_array(snapshot.fetch(:teams).map { |team| team.fetch(:id) }) + expect(follower.beamers.pluck(:sync_source_id)).to match_array(snapshot.fetch(:beamers).map { |beamer| beamer.fetch(:id) }) + end + + it 'ignores stale snapshots that arrive after a newer one' do + follower = create(:tournament, read_only_mode: true, sync_auth_token: 'shared-secret') + + newer_source = create(:tournament, name: 'Newer name') + older_source = create(:tournament, name: 'Older name') + + newer_snapshot = TournamentSnapshotBuilder.build(newer_source).merge(sync_generated_at: '2026-04-14T10:30:02Z') + older_snapshot = TournamentSnapshotBuilder.build(older_source).merge(sync_generated_at: '2026-04-14T10:30:01Z') + + described_class.import!(tournament: follower, snapshot: newer_snapshot) + described_class.import!(tournament: follower, snapshot: older_snapshot) + + follower.reload + expect(follower.name).to eq('Newer name') + expect(follower.sync_last_imported_snapshot_at).to eq(Time.zone.parse('2026-04-14T10:30:02Z')) + end + + it 'round-trips large leader tournament graph with mismatched ids, many groups, deciders and 64-team playoffs' do + create_list(:tournament, 3) + source = create_large_source_tournament! + create_list(:tournament, 5) + follower = create(:tournament, read_only_mode: true, sync_auth_token: 'shared-secret') + + snapshot = TournamentSnapshotBuilder.build(source) + + described_class.import!(tournament: follower, snapshot: snapshot) + + follower.reload + expect(follower.id).not_to eq(source.id) + expect(sync_signature(follower)).to eq(sync_signature(source)) + expect(follower.teams.count).to eq(128) + expect(follower.stages.where(level: -1).first.groups.count).to eq(32) + expect(follower.stages.where('level >= 0').sum { |stage| stage.matches.count }).to eq(63) + expect(follower.matches.select(&:decider?).count).to eq(source.matches.select(&:decider?).count) + expect(follower.teams.pluck(:sync_source_id)).to match_array(source.teams.pluck(:id)) + expect(source.teams.pluck(:id) & follower.teams.pluck(:id)).not_to match_array(source.teams.pluck(:id)) + end + end + + def create_large_source_tournament! + tournament = create( + :tournament, + teams_count: 0, + name: 'Large Sync Source', + playoff_teams_amount: 64, + instant_finalists_amount: 64, + intermediate_round_participants_amount: 0 + ) + tournament.beamers.find_by!(name: Beamer::DEFAULT_NAME).update!( + display_state: 'groups', + qr_code_url: 'https://example.com/default-qr', + logo_url: '/logos/default.png' + ) + tournament.beamers.create!( + name: 'stream-a', + display_state: 'during_game', + qr_code_url: 'https://example.com/stream-a', + logo_url: '/logos/stream-a.png' + ) + tournament.beamers.create!( + name: 'stream-b', + display_state: 'between_games', + qr_code_url: 'https://example.com/stream-b', + logo_url: '/logos/stream-b.png' + ) + + group_stage = Stage.create!(tournament: tournament, level: -1, state: :finished) + advancing_teams = [] + + 32.times do |group_index| + group = Group.create!(stage: group_stage, number: group_index + 1) + teams = 4.times.map do |seed_index| + create( + :team, + tournament: tournament, + name: "Leader-G#{group_index + 1}-S#{seed_index + 1}" + ) + end + + teams[0].update!(name: "#{teams[0].name} Renamed") if group_index.zero? + teams[1].update!(name: "#{teams[1].name} Renamed") if group_index == 15 + + create_group_round_robin!(group: group, teams: teams) + create_group_scores!(group: group, teams: teams) + + if group_index < 3 + create_decider_match!(group: group, teams: teams.take(2), hidden_points: [8 - group_index, 4 + group_index]) + end + + advancing_teams.concat(teams.take(2)) + end + + result = AddPlayoffsToTournamentAndSave.call(tournament: tournament, teams: advancing_teams) + expect(result).to be_a_success + + tournament.reload + play_some_playoff_matches!(tournament) + tournament.reload + end + + def create_group_round_robin!(group:, teams:) + pairings = [ + [0, 1, 2, 1], + [2, 3, 3, 0], + [0, 2, 4, 1], + [1, 3, 2, 0], + [0, 3, 5, 2], + [1, 2, 4, 3] + ] + + pairings.each_with_index do |(left_index, right_index, left_points, right_points), position| + match = Match.create!(group: group, state: :finished, position: position, decider: false) + MatchScore.create!(match: match, team: teams[left_index], points: left_points, hidden_points: 0) + MatchScore.create!(match: match, team: teams[right_index], points: right_points, hidden_points: 0) + end + end + + def create_decider_match!(group:, teams:, hidden_points:) + match = Match.create!(group: group, state: :finished, position: 6, decider: true) + MatchScore.create!(match: match, team: teams.first, points: 0, hidden_points: hidden_points.first) + MatchScore.create!(match: match, team: teams.second, points: 0, hidden_points: hidden_points.last) + end + + def create_group_scores!(group:, teams:) + teams.each_with_index do |team, index| + GroupScore.create!( + group: group, + team: team, + group_points: 9 - index, + scored_points: 12 - index, + received_points: index, + position: index + 1 + ) + end + end + + def play_some_playoff_matches!(tournament) + initial_stage = tournament.stages.where('level >= 0').max_by(&:level) + initial_stage.matches.sort_by(&:position).first(4).each_with_index do |match, index| + match.update!(state: :finished) + ordered_scores = match.match_scores.sort_by(&:id) + ordered_scores.first.update!(points: 20 + index) + ordered_scores.second.update!(points: index) + PlayoffStageService.populate_match_below(match).uniq.each(&:save!) + end + end + + def sync_signature(tournament) + { + name: tournament.name, + public: tournament.public, + playoff_teams_amount: tournament.playoff_teams_amount, + instant_finalists_amount: tournament.instant_finalists_amount, + intermediate_round_participants_amount: tournament.intermediate_round_participants_amount, + timer_end: tournament.timer_end&.iso8601, + teams: tournament.teams.order(:name).map { |team| team.name }, + beamers: tournament.beamers.order(:name).map do |beamer| + beamer.slice(:name, :display_state, :is_default, :qr_code_url, :logo_url).symbolize_keys + end, + stages: tournament.stages.order(level: :asc).map do |stage| + { + level: stage.level, + state: stage.state, + groups: stage.groups.order(:number).map do |group| + { + number: group.number, + matches: group.matches.order(:position).map { |match| match_signature(match) }, + group_scores: group.group_scores.includes(:team).sort_by(&:position).map do |score| + { + team_name: score.team.name, + position: score.position, + group_points: score.group_points, + scored_points: score.scored_points, + received_points: score.received_points + } + end + } + end, + matches: stage.matches.order(:position).map { |match| match_signature(match) } + } + end + } + end + + def match_signature(match) + { + state: match.state, + position: match.position, + decider: match.decider, + scores: match.match_scores.includes(:team).sort_by { |score| score.team.name }.map do |score| + { + team_name: score.team.name, + points: score.points, + hidden_points: score.hidden_points + } + end + } + end +end diff --git a/spec/services/tournament_sync_enqueue_spec.rb b/spec/services/tournament_sync_enqueue_spec.rb new file mode 100644 index 0000000..77cc3ad --- /dev/null +++ b/spec/services/tournament_sync_enqueue_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TournamentSyncEnqueue do + describe '.call' do + it 'resets retry state for newly enqueued snapshots' do + tournament = create(:tournament, + sync_target_url: 'https://remote.example.com/tournaments/1/sync_state', + sync_auth_token: 'shared-secret') + entry = TournamentSyncQueueEntry.create!( + tournament: tournament, + snapshot_json: '{"old":true}', + status: 'pending', + attempts: 4, + last_error: 'network down', + last_attempt_at: 5.minutes.ago, + next_attempt_at: 3.minutes.from_now, + locked_at: 1.minute.ago, + lock_token: 'stale-lock' + ) + + allow(TournamentSyncWorker).to receive(:start) + allow(TournamentSyncWorker).to receive(:wake) + + described_class.call(tournament) + + entry.reload + expect(entry.status).to eq('pending') + expect(entry.attempts).to eq(0) + expect(entry.last_error).to be_nil + expect(entry.last_attempt_at).to be_nil + expect(entry.next_attempt_at).to be <= Time.current + expect(entry.locked_at).to be_nil + expect(entry.lock_token).to be_nil + end + end +end diff --git a/spec/services/tournament_sync_processor_spec.rb b/spec/services/tournament_sync_processor_spec.rb new file mode 100644 index 0000000..f3b54ec --- /dev/null +++ b/spec/services/tournament_sync_processor_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TournamentSyncProcessor do + describe '.process_due!' do + it 'retries failed pushes asynchronously from queue state' do + tournament = create(:tournament, + sync_target_url: 'https://remote.example.com/tournaments/1/sync_state', + sync_auth_token: 'shared-secret') + TournamentSyncEnqueue.call(tournament) + entry = TournamentSyncQueueEntry.find_by!(tournament: tournament) + + allow(TournamentSyncPusher).to receive(:push_snapshot!).and_raise(TournamentSyncPusher::SyncFailed, 'network down') + + described_class.process_due! + + entry.reload + expect(entry.status).to eq('pending') + expect(entry.last_error).to eq('network down') + expect(entry.attempts).to eq(1) + expect(entry.next_attempt_at).to be > Time.current + end + end +end diff --git a/spec/services/tournament_sync_schema_spec.rb b/spec/services/tournament_sync_schema_spec.rb new file mode 100644 index 0000000..de7a8f1 --- /dev/null +++ b/spec/services/tournament_sync_schema_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TournamentSyncSchema do + describe 'column coverage' do + described_class::MODEL_COLUMNS.each_key do |model_class| + it "accounts for every #{model_class.name} column in sync contract" do + synced_columns = described_class.synced_columns_for(model_class) + ignored_columns = described_class.ignored_columns_for(model_class) + accounted_columns = synced_columns + ignored_columns + + expect(accounted_columns.uniq).to match_array(model_class.column_names) + expect(synced_columns & ignored_columns).to be_empty + end + end + end +end diff --git a/tasks.py b/tasks.py index fe96111..ed68ca1 100644 --- a/tasks.py +++ b/tasks.py @@ -17,6 +17,10 @@ SERVER_PORT = "3000" TEST_BASE_URL = f"http://127.0.0.1:{SERVER_PORT}" BLACKBOX_BASE_URL = TEST_BASE_URL BLACKBOX_INTERNAL_BASE_URL = "http://app:3000" +BLACKBOX_SOURCE_INTERNAL_BASE_URL = "http://source-app:3000" +BLACKBOX_FOLLOWER_INTERNAL_BASE_URL = "http://follower-app:3000" +BLACKBOX_SOURCE_HOST_PORT = "3002" +BLACKBOX_FOLLOWER_HOST_PORT = "3003" E2E_EMAIL = "e2e@example.com" E2E_PASSWORD = "password123" @@ -29,8 +33,12 @@ PRODUCTION_TAG = "registry.gitlab.com/turniere/turniere-backend/production/commi TEST_TAG = "registry.gitlab.com/turniere/turniere-backend/test/commits:local" BLACKBOX_COMPOSE_FILE = "docker-compose.blackbox.yml" BLACKBOX_PROJECT = "turniere-blackbox" +BLACKBOX_FOLLOW_COMPOSE_FILE = "docker-compose.blackbox-follow.yml" +BLACKBOX_FOLLOW_PROJECT = "turniere-blackbox-follow" BLACKBOX_POSTGRES_IMAGE = "postgres:16" BLACKBOX_DB_NAME = "turniere_blackbox" +BLACKBOX_SOURCE_DB_NAME = "turniere_source_blackbox" +BLACKBOX_FOLLOWER_DB_NAME = "turniere_follower_blackbox" BLACKBOX_DB_USER = "turniere" BLACKBOX_DB_PASSWORD = "turniere" BLACKBOX_SECRET_KEY_BASE = "turniere-blackbox-secret-key-base-please-change-in-real-prod" @@ -144,7 +152,7 @@ def _stream_subprocess(command, env=None, check=True): output_chunks = [] assert process.stdout is not None for line in process.stdout: - print(line, end="") + print(line, end="", flush=True) output_chunks.append(line) process.wait() @@ -185,6 +193,37 @@ def _compose_env( ) +def _follow_compose_env( + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, +): + return _env( + TURNIERE_BLACKBOX_SOURCE_HOST_PORT=source_host_port, + TURNIERE_BLACKBOX_FOLLOWER_HOST_PORT=follower_host_port, + TURNIERE_BLACKBOX_APP_IMAGE=app_image, + TURNIERE_BLACKBOX_RUNNER_IMAGE=runner_image, + TURNIERE_BLACKBOX_POSTGRES_IMAGE=postgres_image, + TURNIERE_BLACKBOX_SOURCE_POSTGRES_DB=BLACKBOX_SOURCE_DB_NAME, + TURNIERE_BLACKBOX_FOLLOWER_POSTGRES_DB=BLACKBOX_FOLLOWER_DB_NAME, + TURNIERE_BLACKBOX_POSTGRES_USER=BLACKBOX_DB_USER, + TURNIERE_BLACKBOX_POSTGRES_PASSWORD=BLACKBOX_DB_PASSWORD, + TURNIERE_BLACKBOX_SECRET_KEY_BASE=BLACKBOX_SECRET_KEY_BASE, + TURNIERE_BLACKBOX_MAILGUN_API_KEY=BLACKBOX_MAILGUN_API_KEY, + TURNIERE_BLACKBOX_MAILGUN_DOMAIN=BLACKBOX_MAILGUN_DOMAIN, + TURNIERE_SOURCE_E2E_BASE_URL=BLACKBOX_SOURCE_INTERNAL_BASE_URL, + TURNIERE_FOLLOWER_E2E_BASE_URL=BLACKBOX_FOLLOWER_INTERNAL_BASE_URL, + TURNIERE_E2E_EMAIL=E2E_EMAIL, + TURNIERE_E2E_PASSWORD=E2E_PASSWORD, + TURNIERE_E2E_USERNAME=E2E_USERNAME, + TURNIERE_E2E_ALT_EMAIL=E2E_ALT_EMAIL, + TURNIERE_E2E_ALT_PASSWORD=E2E_ALT_PASSWORD, + TURNIERE_E2E_ALT_USERNAME=E2E_ALT_USERNAME, + ) + + def _format_command(command): return " ".join(shlex.quote(part) for part in command) @@ -198,13 +237,55 @@ def _rspec_command(*paths): return command -def _compose_run(compose_args, env, capture_output=False, check=True): - command = _docker_compose_cmd() + compose_args +def _compose_run(compose_args, env, capture_output=False, check=True, project_name=BLACKBOX_PROJECT, compose_file=BLACKBOX_COMPOSE_FILE): + command = _docker_compose_cmd(project_name=project_name, compose_file=compose_file) + compose_args return _run_subprocess(command, env=env, capture_output=capture_output, check=check) def _print_header(title): - print(f"\n== {title} ==") + print(f"\n== {title} ==", flush=True) + + +def _blackbox_network_smoke(c, target_base_url): + command = [ + "ruby", + "-rnet/http", + "-ruri", + "-e", + ( + "uri = URI.parse(ARGV.fetch(0) + '/healthz'); " + "res = Net::HTTP.get_response(uri); " + "abort(\"healthcheck #{res.code}\") unless res.is_a?(Net::HTTPSuccess); " + "puts(\"network smoke ok #{uri}\")" + ), + target_base_url, + ] + c.run(_format_command(command), pty=True) + + +def _blackbox_service_to_service_smoke(c, env, source_label, target_base_url): + command = [ + "bundle", + "exec", + "rails", + "runner", + ( + "require 'net/http'; " + "require 'uri'; " + "uri = URI.parse(ENV.fetch('BLACKBOX_SMOKE_TARGET_URL') + '/healthz'); " + "Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 3, read_timeout: 3) do |http| " + "res = http.get(uri.request_uri); " + "abort(\"service smoke #{res.code}\") unless res.is_a?(Net::HTTPSuccess); " + "puts(\"service smoke ok #{ENV.fetch('BLACKBOX_SMOKE_SOURCE_LABEL')} -> #{uri}\"); " + "end" + ), + ] + smoke_env = env.copy() + smoke_env.update( + BLACKBOX_SMOKE_SOURCE_LABEL=source_label, + BLACKBOX_SMOKE_TARGET_URL=target_base_url, + ) + c.run(_format_command(command), env=smoke_env, pty=True) def _bootstrap_user_command(email, password, username): @@ -305,8 +386,22 @@ def _shared_production_env(): ) +def _production_env_for(db_host, db_name): + return _env( + RAILS_ENV="production", + POSTGRES_URL=f"postgres://{BLACKBOX_DB_USER}:{BLACKBOX_DB_PASSWORD}@{db_host}:5432/{db_name}", + POSTGRES_USERNAME=BLACKBOX_DB_USER, + POSTGRES_PASSWORD=BLACKBOX_DB_PASSWORD, + SECRET_KEY_BASE=BLACKBOX_SECRET_KEY_BASE, + MAILGUN_API_KEY=BLACKBOX_MAILGUN_API_KEY, + MAILGUN_DOMAIN=BLACKBOX_MAILGUN_DOMAIN, + RAILS_LOG_TO_STDOUT="1", + RAILS_SERVE_STATIC_FILES="1", + ) + + def _run_blackbox_rspec(base_url, email, password, username, alt_email=E2E_ALT_EMAIL, alt_password=E2E_ALT_PASSWORD, - alt_username=E2E_ALT_USERNAME): + alt_username=E2E_ALT_USERNAME, spec_path="spec/e2e/http", extra_env=None): env = _env( TURNIERE_E2E_BASE_URL=base_url, TURNIERE_E2E_EXPECT_PROFILING="false", @@ -317,8 +412,10 @@ def _run_blackbox_rspec(base_url, email, password, username, alt_email=E2E_ALT_E TURNIERE_E2E_ALT_PASSWORD=alt_password, TURNIERE_E2E_ALT_USERNAME=alt_username, ) - command = _rspec_command("spec/e2e/http") - print(_format_command(command)) + if extra_env: + env.update(extra_env) + command = _rspec_command(spec_path) + print(_format_command(command), flush=True) result = _stream_subprocess(command, env=env, check=False) report = _parse_rspec_report(f"{result.stdout}\n{result.stderr}") @@ -362,6 +459,19 @@ def _stop_process(process, log_handle): log_handle.close() +def _bootstrap_blackbox_users(c, env, email, password, username, alt_email, alt_password, alt_username): + c.run( + _bootstrap_user_command(email, password, username), + env=env, + pty=True, + ) + c.run( + _bootstrap_user_command(alt_email, alt_password, alt_username), + env=env, + pty=True, + ) + + @task def setup(c): """Install the pinned Bundler and local backend gem dependencies.""" @@ -836,6 +946,653 @@ def docker_blackbox_test( ) +@task(name="docker-blackbox-follow-up") +def docker_blackbox_follow_up( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, + build=True, +): + """Build Docker images, then boot source/follower Postgres and production apps for follower blackbox testing.""" + if build: + docker_build_all(c) + + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + + _print_header("Starting follower production blackbox stack") + _compose_run( + ["down", "-v", "--remove-orphans"], + env=env, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + _compose_run( + ["up", "-d", "source-postgres", "follower-postgres", "source-app", "follower-app"], + env=env, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + + source_base_url = f"http://127.0.0.1:{source_host_port}" + follower_base_url = f"http://127.0.0.1:{follower_host_port}" + _print_header(f"Waiting for source app healthcheck at {source_base_url}") + _wait_for_http(base_url=source_base_url, timeout=120) + + _print_header(f"Waiting for follower app healthcheck at {follower_base_url}") + _wait_for_http(base_url=follower_base_url, timeout=120) + + print("Follower production apps ready for blackbox E2E.") + + +@task(name="docker-blackbox-follow-manual-up") +def docker_blackbox_follow_manual_up( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, + build=True, +): + """Boot local leader/follower production setup for manual testing and print next steps.""" + docker_blackbox_follow_up( + c, + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + build=build, + ) + + source_env = _production_env_for("source-postgres", BLACKBOX_SOURCE_DB_NAME) + follower_env = _production_env_for("follower-postgres", BLACKBOX_FOLLOWER_DB_NAME) + + _print_header("Running source production database migrations") + c.run("bundle exec rails db:migrate", env=source_env, pty=True) + + _print_header("Running follower production database migrations") + c.run("bundle exec rails db:migrate", env=follower_env, pty=True) + + _print_header("Bootstrapping confirmed E2E users on source and follower") + _bootstrap_blackbox_users(c, source_env, E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME, E2E_ALT_EMAIL, E2E_ALT_PASSWORD, E2E_ALT_USERNAME) + _bootstrap_blackbox_users(c, follower_env, E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME, E2E_ALT_EMAIL, E2E_ALT_PASSWORD, E2E_ALT_USERNAME) + + source_base_url = f"http://127.0.0.1:{source_host_port}" + follower_base_url = f"http://127.0.0.1:{follower_host_port}" + _print_header("Leader/follower manual setup ready") + print(f"Leader URL: {source_base_url}") + print(f"Follower URL: {follower_base_url}") + print(f"E2E owner email: {E2E_EMAIL}") + print(f"E2E owner password: {E2E_PASSWORD}") + print("Next steps:") + print("1. Create read-only follower tournament on follower backend.") + print("2. Create normal leader tournament on leader backend.") + print("3. Configure leader sync_target_url to follower /sync_state endpoint with shared token.") + print("4. Point frontend local leader view at leader URL and remote view at follower URL.") + print("5. Tear down with: inv docker-blackbox-follow-down") + + +@task(name="docker-blackbox-follow-down") +def docker_blackbox_follow_down( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, +): + """Stop and remove follower production blackbox stack.""" + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + _compose_run( + ["down", "-v", "--remove-orphans"], + env=env, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + + +@task(name="docker-blackbox-follow-test") +def docker_blackbox_follow_test( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, +): + """Run follower-sync HTTP E2E suite against running source/follower production blackbox stack.""" + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + + _print_header("Running follower sync HTTP E2E against production images") + command = [ + "run", + "--rm", + "e2e", + "inv", + "blackbox-follow-service", + ] + full_command = _docker_compose_cmd(project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE) + command + print(_format_command(full_command), flush=True) + result = _run_subprocess_live(full_command, env=env, check=False) + + if result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, + result.args, + output=result.stdout, + stderr=result.stderr, + ) + + +@task(name="docker-blackbox-follow-bootstrap-users") +def docker_blackbox_follow_bootstrap_users( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, +): + """Bootstrap confirmed E2E users in local source/follower production blackbox DBs.""" + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + + for app_service in ("source-app", "follower-app"): + _print_header(f"Bootstrapping confirmed E2E users in {app_service}") + for bootstrap_email, bootstrap_password, bootstrap_username in ( + (E2E_EMAIL, E2E_PASSWORD, E2E_USERNAME), + (E2E_ALT_EMAIL, E2E_ALT_PASSWORD, E2E_ALT_USERNAME), + ): + bootstrap = _compose_run( + ["exec", "-T", app_service, "bash", "-lc", _bootstrap_user_command(bootstrap_email, bootstrap_password, bootstrap_username)], + env=env, + capture_output=True, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + if bootstrap.returncode != 0: + print(bootstrap.stdout or "", end="") + print(bootstrap.stderr or "", end="") + raise subprocess.CalledProcessError( + bootstrap.returncode, + bootstrap.args, + output=bootstrap.stdout, + stderr=bootstrap.stderr, + ) + + +@task(name="docker-blackbox-follow-debug-auth") +def docker_blackbox_follow_debug_auth( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, +): + """Debug source/follower login and first authenticated create inside local follower blackbox runner.""" + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + + script = """ +require_relative 'e2e/lib/api_client' + +def inspect_client(base_url) + client = TurniereE2E::ApiClient.new(base_url: base_url) + login = client.login!(email: ENV.fetch('TURNIERE_E2E_EMAIL'), password: ENV.fetch('TURNIERE_E2E_PASSWORD')) + puts({ base_url: base_url, login_status: login[:status], login_headers: login[:headers].slice('access-token', 'client', 'uid', 'expiry', 'token-type', 'authorization'), authenticated: client.authenticated? }.inspect) + + create = client.post('/tournaments', body: { + name: "Debug #{base_url}", + description: 'Debug tournament', + public: false, + teams: [ + { name: 'One' }, + { name: 'Two' }, + { name: 'Three' }, + { name: 'Four' } + ] + }) + puts({ base_url: base_url, create_status: create[:status], create_json: create[:json], create_headers: create[:headers].slice('access-token', 'client', 'uid', 'expiry', 'token-type', 'authorization') }.inspect) +end + +inspect_client(ENV.fetch('TURNIERE_SOURCE_E2E_BASE_URL')) +inspect_client(ENV.fetch('TURNIERE_FOLLOWER_E2E_BASE_URL')) +""" + command = [ + "run", + "--rm", + "e2e", + "ruby", + "-e", + script, + ] + print(_format_command(_docker_compose_cmd(project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE) + command)) + result = _compose_run( + command, + env=env, + capture_output=True, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + if result.stdout: + print(result.stdout.rstrip()) + if result.stderr: + print(result.stderr.rstrip()) + + if result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, + result.args, + output=result.stdout, + stderr=result.stderr, + ) + + +@task(name="docker-blackbox-follow-debug-seed") +def docker_blackbox_follow_debug_seed( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, +): + """Debug repeated mixed tournament creation inside local follower blackbox runner.""" + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + + script = """ +require_relative 'e2e/lib/api_client' + +def create_playoff_payload(name) + { + name: name, + description: 'Debug playoff tournament', + public: false, + teams: [ + { name: 'One' }, + { name: 'Two' }, + { name: 'Three' }, + { name: 'Four' } + ] + } +end + +def create_group_payload(name) + { + name: name, + description: 'Debug group tournament', + public: true, + group_stage: true, + playoff_teams_amount: 8, + teams: 4.times.flat_map do |group_index| + 4.times.map do |seed_index| + { name: "#{name}-G#{group_index + 1}-S#{seed_index + 1}", group: group_index } + end + end + } +end + +def inspect_seed(base_url) + client = TurniereE2E::ApiClient.new(base_url: base_url) + login = client.login!(email: ENV.fetch('TURNIERE_E2E_EMAIL'), password: ENV.fetch('TURNIERE_E2E_PASSWORD')) + puts({ base_url: base_url, login_status: login[:status], authenticated: client.authenticated? }.inspect) + + [ + ['playoff-1', create_playoff_payload('Debug Playoff 1')], + ['group-1', create_group_payload('Debug Group 1')], + ['playoff-2', create_playoff_payload('Debug Playoff 2')], + ['group-2', create_group_payload('Debug Group 2')] + ].each do |label, payload| + response = client.post('/tournaments', body: payload) + puts({ + base_url: base_url, + step: label, + status: response[:status], + json: response[:json], + headers: response[:headers].slice('access-token', 'client', 'uid', 'expiry', 'token-type', 'authorization'), + authenticated: client.authenticated? + }.inspect) + end +end + +inspect_seed(ENV.fetch('TURNIERE_SOURCE_E2E_BASE_URL')) +inspect_seed(ENV.fetch('TURNIERE_FOLLOWER_E2E_BASE_URL')) +""" + command = [ + "run", + "--rm", + "e2e", + "ruby", + "-e", + script, + ] + print(_format_command(_docker_compose_cmd(project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE) + command)) + result = _compose_run( + command, + env=env, + capture_output=True, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + if result.stdout: + print(result.stdout.rstrip()) + if result.stderr: + print(result.stderr.rstrip()) + + if result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, + result.args, + output=result.stdout, + stderr=result.stderr, + ) + + +@task(name="docker-blackbox-follow-debug-group-create") +def docker_blackbox_follow_debug_group_create( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, +): + """Debug single fresh-login group-stage tournament creation inside local follower blackbox runner.""" + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + + script = """ +require_relative 'e2e/lib/api_client' + +def create_group_payload(name) + { + name: name, + description: 'Debug group tournament', + public: true, + group_stage: true, + playoff_teams_amount: 8, + teams: 4.times.flat_map do |group_index| + 4.times.map do |seed_index| + { name: "#{name}-G#{group_index + 1}-S#{seed_index + 1}", group: group_index } + end + end + } +end + +def inspect_group_create(base_url) + client = TurniereE2E::ApiClient.new(base_url: base_url) + login = client.login!(email: ENV.fetch('TURNIERE_E2E_EMAIL'), password: ENV.fetch('TURNIERE_E2E_PASSWORD')) + puts({ base_url: base_url, login_status: login[:status], authenticated: client.authenticated? }.inspect) + response = client.post('/tournaments', body: create_group_payload('Fresh Debug Group')) + puts({ base_url: base_url, create_status: response[:status], create_json: response[:json], create_headers: response[:headers].slice('access-token', 'client', 'uid', 'expiry', 'token-type', 'authorization') }.inspect) +end + +inspect_group_create(ENV.fetch('TURNIERE_SOURCE_E2E_BASE_URL')) +inspect_group_create(ENV.fetch('TURNIERE_FOLLOWER_E2E_BASE_URL')) +""" + command = [ + "run", + "--rm", + "e2e", + "ruby", + "-e", + script, + ] + print(_format_command(_docker_compose_cmd(project_name=BLACKBOX_FOLLOW_PROJECT, compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE) + command)) + result = _compose_run( + command, + env=env, + capture_output=True, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + if result.stdout: + print(result.stdout.rstrip()) + if result.stderr: + print(result.stderr.rstrip()) + + if result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, + result.args, + output=result.stdout, + stderr=result.stderr, + ) + + +@task(name="docker-blackbox-follow-debug-sync") +def docker_blackbox_follow_debug_sync( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, + log_tail=120, +): + """Print local follower blackbox sync queue state and recent app logs.""" + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + + script = """ +puts '== Source tournaments ==' +Tournament.order(:id).pluck(:id, :name, :read_only_mode, :sync_target_url, :sync_last_pushed_at, :sync_last_push_error).each { |row| p row } +puts '== Source sync queue ==' +TournamentSyncQueueEntry.order(:id).pluck(:id, :tournament_id, :status, :attempts, :last_error, :next_attempt_at, :last_attempt_at).each { |row| p row } +puts '== Follower tournaments ==' +Tournament.order(:id).pluck(:id, :name, :read_only_mode, :sync_target_url, :sync_last_pushed_at, :sync_last_push_error).each { |row| p row } +""" + + for service, db_host in (("source-app", "source-postgres"), ("follower-app", "follower-postgres")): + _print_header(f"Sync state from {service}") + db_name = BLACKBOX_SOURCE_DB_NAME if service == "source-app" else BLACKBOX_FOLLOWER_DB_NAME + service_env = _production_env_for(db_host, db_name) + runner = _compose_run( + ["run", "--rm", "-e", f"POSTGRES_URL={service_env['POSTGRES_URL']}", "-e", f"POSTGRES_USERNAME={service_env['POSTGRES_USERNAME']}", "-e", f"POSTGRES_PASSWORD={service_env['POSTGRES_PASSWORD']}", "-e", f"SECRET_KEY_BASE={service_env['SECRET_KEY_BASE']}", "-e", f"MAILGUN_API_KEY={service_env['MAILGUN_API_KEY']}", "-e", f"MAILGUN_DOMAIN={service_env['MAILGUN_DOMAIN']}", "-e", f"RAILS_LOG_TO_STDOUT={service_env['RAILS_LOG_TO_STDOUT']}", "-e", f"RAILS_SERVE_STATIC_FILES={service_env['RAILS_SERVE_STATIC_FILES']}", "-e", "RAILS_ENV=production", "e2e", "bundle", "exec", "rails", "runner", script], + env=env, + capture_output=True, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + if runner.stdout: + print(runner.stdout.rstrip()) + if runner.stderr: + print(runner.stderr.rstrip()) + if runner.returncode != 0: + raise subprocess.CalledProcessError( + runner.returncode, + runner.args, + output=runner.stdout, + stderr=runner.stderr, + ) + + _print_header(f"Recent logs from {service}") + logs = _compose_run( + ["logs", "--tail", str(log_tail), service], + env=env, + capture_output=True, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + output = logs.stdout or logs.stderr or "" + print(output.rstrip()) + + +@task(name="docker-blackbox-follow-debug-compare") +def docker_blackbox_follow_debug_compare( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, +): + """Compare latest leader tournament and follower read-only tournament via HTTP API.""" + env = _follow_compose_env( + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + + script = """ +def signature(tournament) + { + id: tournament.id, + name: tournament.name, + public: tournament.public, + playoff_teams_amount: tournament.playoff_teams_amount, + instant_finalists_amount: tournament.instant_finalists_amount, + intermediate_round_participants_amount: tournament.intermediate_round_participants_amount, + sync_source_tournament_id: tournament.respond_to?(:sync_source_tournament_id) ? tournament.sync_source_tournament_id : nil, + team_names: tournament.teams.order(:id).pluck(:name), + stage_levels: tournament.stages.order(:level).map { |stage| [stage.level, stage.state, stage.groups.count, stage.matches.count] }, + total_matches: tournament.matches.count, + decider_matches: tournament.matches.count(&:decider?) + } +end + +tournament = if ENV['FOLLOW_COMPARE_MODE'] == 'leader' + Tournament.where.not(sync_target_url: nil).order(:id).last + else + Tournament.where(read_only_mode: true).order(:id).last + end + +puts signature(tournament).inspect +""" + + for mode, db_host, db_name in (("leader", "source-postgres", BLACKBOX_SOURCE_DB_NAME), ("follower", "follower-postgres", BLACKBOX_FOLLOWER_DB_NAME)): + _print_header(f"Compare {mode} tournament") + service_env = _production_env_for(db_host, db_name) + runner = _compose_run( + ["run", "--rm", "-e", f"POSTGRES_URL={service_env['POSTGRES_URL']}", "-e", f"POSTGRES_USERNAME={service_env['POSTGRES_USERNAME']}", "-e", f"POSTGRES_PASSWORD={service_env['POSTGRES_PASSWORD']}", "-e", f"SECRET_KEY_BASE={service_env['SECRET_KEY_BASE']}", "-e", f"MAILGUN_API_KEY={service_env['MAILGUN_API_KEY']}", "-e", f"MAILGUN_DOMAIN={service_env['MAILGUN_DOMAIN']}", "-e", f"RAILS_LOG_TO_STDOUT={service_env['RAILS_LOG_TO_STDOUT']}", "-e", f"RAILS_SERVE_STATIC_FILES={service_env['RAILS_SERVE_STATIC_FILES']}", "-e", "RAILS_ENV=production", "-e", f"FOLLOW_COMPARE_MODE={mode}", "e2e", "bundle", "exec", "rails", "runner", script], + env=env, + capture_output=True, + check=False, + project_name=BLACKBOX_FOLLOW_PROJECT, + compose_file=BLACKBOX_FOLLOW_COMPOSE_FILE, + ) + if runner.stdout: + print(runner.stdout.rstrip()) + if runner.stderr: + print(runner.stderr.rstrip()) + if runner.returncode != 0: + raise subprocess.CalledProcessError( + runner.returncode, + runner.args, + output=runner.stdout, + stderr=runner.stderr, + ) + + +@task(name="docker-blackbox-follow-run") +def docker_blackbox_follow_run( + c, + source_host_port=BLACKBOX_SOURCE_HOST_PORT, + follower_host_port=BLACKBOX_FOLLOWER_HOST_PORT, + app_image=PRODUCTION_TAG, + runner_image=TEST_TAG, + postgres_image=BLACKBOX_POSTGRES_IMAGE, + build=True, + keep_running=False, +): + """Build, boot, run follower-sync blackbox E2E, then tear down unless keep_running=true.""" + exit_error = None + try: + docker_blackbox_follow_up( + c, + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + build=build, + ) + docker_blackbox_follow_test( + c, + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + except subprocess.CalledProcessError as error: + exit_error = error + raise + finally: + if keep_running: + print("Keeping follower production blackbox stack running.") + else: + docker_blackbox_follow_down( + c, + source_host_port=source_host_port, + follower_host_port=follower_host_port, + app_image=app_image, + runner_image=runner_image, + postgres_image=postgres_image, + ) + if exit_error is not None: + print("Follower blackbox run failed.") + + @task(name="blackbox-service") def blackbox_service( c, @@ -855,21 +1612,67 @@ def blackbox_service( _wait_for_http(base_url=base_url, timeout=120) _print_header("Bootstrapping confirmed E2E user") - c.run( - _bootstrap_user_command(email, password, username), - env=_shared_production_env(), - pty=True, - ) - c.run( - _bootstrap_user_command(alt_email, alt_password, alt_username), - env=_shared_production_env(), - pty=True, - ) + _bootstrap_blackbox_users(c, _shared_production_env(), email, password, username, alt_email, alt_password, alt_username) _print_header("Running HTTP E2E against production image") _run_blackbox_rspec(base_url, email, password, username, alt_email, alt_password, alt_username) +@task(name="blackbox-follow-service") +def blackbox_follow_service( + c, + source_base_url=BLACKBOX_SOURCE_INTERNAL_BASE_URL, + follower_base_url=BLACKBOX_FOLLOWER_INTERNAL_BASE_URL, + email=E2E_EMAIL, + password=E2E_PASSWORD, + username=E2E_USERNAME, + alt_email=E2E_ALT_EMAIL, + alt_password=E2E_ALT_PASSWORD, + alt_username=E2E_ALT_USERNAME, +): + """Run follower-sync HTTP E2E against two independent production app services.""" + source_env = _production_env_for("source-postgres", BLACKBOX_SOURCE_DB_NAME) + follower_env = _production_env_for("follower-postgres", BLACKBOX_FOLLOWER_DB_NAME) + + _print_header("Running source production database migrations") + c.run("bundle exec rails db:migrate", env=source_env, pty=True) + + _print_header("Running follower production database migrations") + c.run("bundle exec rails db:migrate", env=follower_env, pty=True) + + _print_header(f"Waiting for source app healthcheck at {source_base_url}") + _wait_for_http(base_url=source_base_url, timeout=120) + + _print_header(f"Waiting for follower app healthcheck at {follower_base_url}") + _wait_for_http(base_url=follower_base_url, timeout=120) + + _print_header("Running source/follower network smoke") + _blackbox_network_smoke(c, source_base_url) + _blackbox_network_smoke(c, follower_base_url) + _blackbox_service_to_service_smoke(c, source_env, "source-app", follower_base_url) + _blackbox_service_to_service_smoke(c, follower_env, "follower-app", source_base_url) + + _print_header("Bootstrapping confirmed E2E users on source and follower") + _bootstrap_blackbox_users(c, source_env, email, password, username, alt_email, alt_password, alt_username) + _bootstrap_blackbox_users(c, follower_env, email, password, username, alt_email, alt_password, alt_username) + + _print_header("Running follower sync HTTP E2E against production images") + _run_blackbox_rspec( + source_base_url, + email, + password, + username, + alt_email, + alt_password, + alt_username, + spec_path="spec/e2e/http/tournament_follow_sync_spec.rb", + extra_env={ + "TURNIERE_SOURCE_E2E_BASE_URL": source_base_url, + "TURNIERE_FOLLOWER_E2E_BASE_URL": follower_base_url, + }, + ) + + @task(name="blackbox-production") def blackbox_production( c,