feat(metrics): add Prometheus metrics endpoint
This commit is contained in:
parent
3dab5220c2
commit
1bedc87210
|
|
@ -2,5 +2,12 @@
|
|||
|
||||
module ApplicationCable
|
||||
class Connection < ActionCable::Connection::Base
|
||||
def connect
|
||||
MetricsRegistry.action_cable_connected
|
||||
end
|
||||
|
||||
def disconnect
|
||||
MetricsRegistry.action_cable_disconnected
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
class ApplicationController < ActionController::API
|
||||
include DeviseTokenAuth::Concerns::SetUserByToken
|
||||
|
||||
around_action :record_request_metrics
|
||||
before_action :configure_permitted_parameters, if: :devise_controller?
|
||||
|
||||
rescue_from ActionController::ParameterMissing do |e|
|
||||
|
|
@ -19,6 +20,23 @@ class ApplicationController < ActionController::API
|
|||
|
||||
private
|
||||
|
||||
def record_request_metrics
|
||||
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
||||
metrics_status = nil
|
||||
yield
|
||||
rescue StandardError => e
|
||||
metrics_status = ActionDispatch::ExceptionWrapper.status_code_for_exception(e.class.name)
|
||||
raise
|
||||
ensure
|
||||
MetricsRegistry.record_http_request(
|
||||
method: request.request_method,
|
||||
controller: controller_path,
|
||||
action: action_name,
|
||||
status: metrics_status || response.status,
|
||||
duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
|
||||
)
|
||||
end
|
||||
|
||||
def require_owner!(owner)
|
||||
render_forbidden_error if owner != current_user
|
||||
end
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class MetricsController < ApplicationController
|
||||
def index
|
||||
render plain: MetricsRegistry.render_prometheus,
|
||||
content_type: 'text/plain; version=0.0.4'
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class MetricsRegistry
|
||||
HTTP_LABELS = %i[method controller action status].freeze
|
||||
REQUEST_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10].freeze
|
||||
STARTED_AT = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
||||
|
||||
class << self
|
||||
def record_http_request(method:, controller:, action:, status:, duration:)
|
||||
labels = http_labels(method:, controller:, action:, status:)
|
||||
|
||||
synchronize do
|
||||
@http_requests[labels] += 1
|
||||
histogram = @http_request_duration[labels]
|
||||
histogram[:count] += 1
|
||||
histogram[:sum] += duration.to_f
|
||||
REQUEST_DURATION_BUCKETS.each do |bucket|
|
||||
histogram[:buckets][bucket] += 1 if duration.to_f <= bucket
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def action_cable_connected
|
||||
synchronize do
|
||||
@action_cable_connections += 1
|
||||
@action_cable_connections_total += 1
|
||||
end
|
||||
end
|
||||
|
||||
def action_cable_disconnected
|
||||
synchronize do
|
||||
@action_cable_connections = [@action_cable_connections - 1, 0].max
|
||||
end
|
||||
end
|
||||
|
||||
def render_prometheus
|
||||
snapshot = snapshot_metrics
|
||||
|
||||
PrometheusRenderer.new(snapshot).render
|
||||
end
|
||||
|
||||
def reset!
|
||||
@mutex = Mutex.new
|
||||
@http_requests = Hash.new(0)
|
||||
@http_request_duration = Hash.new do |hash, labels|
|
||||
hash[labels] = {
|
||||
count: 0,
|
||||
sum: 0.0,
|
||||
buckets: REQUEST_DURATION_BUCKETS.index_with(0)
|
||||
}
|
||||
end
|
||||
@action_cable_connections = 0
|
||||
@action_cable_connections_total = 0
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def synchronize(&block)
|
||||
reset! if @mutex.nil?
|
||||
|
||||
@mutex.synchronize(&block)
|
||||
end
|
||||
|
||||
def snapshot_metrics
|
||||
synchronize do
|
||||
{
|
||||
http_requests: @http_requests.dup,
|
||||
http_request_duration: @http_request_duration.transform_values do |histogram|
|
||||
{
|
||||
count: histogram.fetch(:count),
|
||||
sum: histogram.fetch(:sum),
|
||||
buckets: histogram.fetch(:buckets).dup
|
||||
}
|
||||
end,
|
||||
action_cable_connections: @action_cable_connections,
|
||||
action_cable_connections_total: @action_cable_connections_total,
|
||||
uptime_seconds: Process.clock_gettime(Process::CLOCK_MONOTONIC) - STARTED_AT,
|
||||
active_record_pool: active_record_pool_stats,
|
||||
git_commit_sha: ENV.fetch('GIT_COMMIT_SHA', 'unknown')
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def http_labels(method:, controller:, action:, status:)
|
||||
{
|
||||
method: method.to_s.upcase,
|
||||
controller: controller.to_s,
|
||||
action: action.to_s,
|
||||
status: status.to_i.to_s
|
||||
}.freeze
|
||||
end
|
||||
|
||||
def active_record_pool_stats
|
||||
return {} unless defined?(ActiveRecord::Base)
|
||||
|
||||
ActiveRecord::Base.connection_pool.stat
|
||||
rescue StandardError
|
||||
{}
|
||||
end
|
||||
end
|
||||
|
||||
class PrometheusRenderer
|
||||
def initialize(snapshot)
|
||||
@snapshot = snapshot
|
||||
@lines = []
|
||||
end
|
||||
|
||||
def render
|
||||
render_build_info
|
||||
render_http_requests
|
||||
render_http_request_duration
|
||||
render_action_cable_connections
|
||||
render_process_metrics
|
||||
render_active_record_pool
|
||||
|
||||
"#{lines.join("\n")}\n"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :snapshot, :lines
|
||||
|
||||
def render_build_info
|
||||
help('turniere_build_info', 'Build metadata for this process.')
|
||||
type('turniere_build_info', 'gauge')
|
||||
sample('turniere_build_info', 1, git_commit_sha: snapshot.fetch(:git_commit_sha))
|
||||
end
|
||||
|
||||
def render_http_requests
|
||||
help('turniere_http_requests_total', 'Total HTTP requests handled by Rails controllers.')
|
||||
type('turniere_http_requests_total', 'counter')
|
||||
snapshot.fetch(:http_requests).each do |labels, count|
|
||||
sample('turniere_http_requests_total', count, labels)
|
||||
end
|
||||
end
|
||||
|
||||
def render_http_request_duration
|
||||
help('turniere_http_request_duration_seconds', 'HTTP request duration in seconds.')
|
||||
type('turniere_http_request_duration_seconds', 'histogram')
|
||||
snapshot.fetch(:http_request_duration).each do |labels, histogram|
|
||||
histogram.fetch(:buckets).each do |bucket, count|
|
||||
sample('turniere_http_request_duration_seconds_bucket', count, labels.merge(le: format_number(bucket)))
|
||||
end
|
||||
sample('turniere_http_request_duration_seconds_bucket', histogram.fetch(:count), labels.merge(le: '+Inf'))
|
||||
sample('turniere_http_request_duration_seconds_sum', histogram.fetch(:sum), labels)
|
||||
sample('turniere_http_request_duration_seconds_count', histogram.fetch(:count), labels)
|
||||
end
|
||||
end
|
||||
|
||||
def render_action_cable_connections
|
||||
help('turniere_action_cable_connections', 'Currently open ActionCable websocket connections.')
|
||||
type('turniere_action_cable_connections', 'gauge')
|
||||
sample('turniere_action_cable_connections', snapshot.fetch(:action_cable_connections))
|
||||
|
||||
help('turniere_action_cable_connections_total', 'Total ActionCable websocket connections opened.')
|
||||
type('turniere_action_cable_connections_total', 'counter')
|
||||
sample('turniere_action_cable_connections_total', snapshot.fetch(:action_cable_connections_total))
|
||||
end
|
||||
|
||||
def render_process_metrics
|
||||
help('turniere_process_uptime_seconds', 'Process uptime in seconds.')
|
||||
type('turniere_process_uptime_seconds', 'gauge')
|
||||
sample('turniere_process_uptime_seconds', snapshot.fetch(:uptime_seconds))
|
||||
|
||||
help('turniere_ruby_threads', 'Ruby threads visible in this process.')
|
||||
type('turniere_ruby_threads', 'gauge')
|
||||
sample('turniere_ruby_threads', Thread.list.count)
|
||||
end
|
||||
|
||||
def render_active_record_pool
|
||||
help('turniere_active_record_connection_pool', 'ActiveRecord connection pool state.')
|
||||
type('turniere_active_record_connection_pool', 'gauge')
|
||||
snapshot.fetch(:active_record_pool).each do |state, value|
|
||||
sample('turniere_active_record_connection_pool', value, state:)
|
||||
end
|
||||
end
|
||||
|
||||
def help(name, text)
|
||||
lines << "# HELP #{name} #{text}"
|
||||
end
|
||||
|
||||
def type(name, type)
|
||||
lines << "# TYPE #{name} #{type}"
|
||||
end
|
||||
|
||||
def sample(name, value, labels = {})
|
||||
lines << "#{name}#{labels_string(labels)} #{format_number(value)}"
|
||||
end
|
||||
|
||||
def labels_string(labels)
|
||||
return '' if labels.empty?
|
||||
|
||||
formatted = labels.map do |key, value|
|
||||
%(#{key}="#{escape_label_value(value)}")
|
||||
end
|
||||
|
||||
"{#{formatted.join(',')}}"
|
||||
end
|
||||
|
||||
def escape_label_value(value)
|
||||
value.to_s.gsub('\\', '\\\\\\').gsub("\n", '\\n').gsub('"', '\"')
|
||||
end
|
||||
|
||||
def format_number(value)
|
||||
return value.to_s if value.is_a?(Integer)
|
||||
|
||||
format('%.12g', value)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
MetricsRegistry.reset!
|
||||
|
|
@ -39,5 +39,6 @@ Rails.application.routes.draw do
|
|||
end
|
||||
resources :match_scores, only: %i[show update]
|
||||
get 'healthz', to: 'health#index'
|
||||
get 'metrics', to: 'metrics#index'
|
||||
get 'version', to: 'version#show'
|
||||
end
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
# Metrics
|
||||
|
||||
The backend exposes Prometheus text metrics at `GET /metrics`.
|
||||
|
||||
The endpoint is unauthenticated so monitoring systems can scrape it directly. It
|
||||
reports per-process in-memory values:
|
||||
|
||||
- `turniere_http_requests_total`
|
||||
- `turniere_http_request_duration_seconds`
|
||||
- `turniere_action_cable_connections`
|
||||
- `turniere_action_cable_connections_total`
|
||||
- `turniere_process_uptime_seconds`
|
||||
- `turniere_ruby_threads`
|
||||
- `turniere_active_record_connection_pool`
|
||||
- `turniere_build_info`
|
||||
|
||||
This covers app-level request and websocket health. Logs, traces, host metrics,
|
||||
and multi-process aggregation should remain infrastructure concerns. If Puma is
|
||||
ever run with multiple worker processes, scrape each process or container replica
|
||||
and aggregate in the monitoring backend.
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'spec_helper'
|
||||
require 'securerandom'
|
||||
require_relative '../../../e2e/lib/action_cable_client'
|
||||
require_relative '../../../e2e/lib/api_client'
|
||||
|
||||
RSpec.describe 'Metrics HTTP E2E' do
|
||||
primary_env_vars = %w[TURNIERE_E2E_EMAIL TURNIERE_E2E_PASSWORD].freeze
|
||||
|
||||
before do
|
||||
missing_vars = primary_env_vars.reject { |name| ENV.key?(name) }
|
||||
skip("HTTP E2E requires #{missing_vars.join(', ')}") unless missing_vars.empty?
|
||||
end
|
||||
|
||||
let(:base_url) { ENV.fetch('TURNIERE_E2E_BASE_URL', 'http://127.0.0.1:3000') }
|
||||
let(:owner_email) { ENV.fetch('TURNIERE_E2E_EMAIL') }
|
||||
let(:owner_password) { ENV.fetch('TURNIERE_E2E_PASSWORD') }
|
||||
let(:anonymous_client) { TurniereE2E::ApiClient.new(base_url:) }
|
||||
|
||||
it 'exposes HTTP request metrics and live websocket connection metrics' do
|
||||
health_response = anonymous_client.get('/healthz')
|
||||
metrics_response = anonymous_client.get('/metrics')
|
||||
|
||||
expect(health_response[:status]).to eq(200)
|
||||
expect(metrics_response[:status]).to eq(200)
|
||||
expect(metrics_response.dig(:headers, 'content-type')).to include('text/plain')
|
||||
|
||||
metrics_body = raw_body(metrics_response)
|
||||
expect(metrics_body).to include('# TYPE turniere_http_requests_total counter')
|
||||
expect(metrics_body).to match(
|
||||
/turniere_http_requests_total\{
|
||||
(?=[^}]*method="GET")
|
||||
(?=[^}]*controller="health")
|
||||
(?=[^}]*action="index")
|
||||
(?=[^}]*status="200")
|
||||
[^}]*
|
||||
\}\ [1-9]\d*/x
|
||||
)
|
||||
expect(metrics_body).to include('# TYPE turniere_http_request_duration_seconds histogram')
|
||||
|
||||
tournament = create_playoff_tournament
|
||||
cable_client = TurniereE2E::ActionCableClient.new(base_url:)
|
||||
cable_client.subscribe_timer!(tournament_id: tournament.fetch(:id))
|
||||
|
||||
expect(metric_value('turniere_action_cable_connections')).to be >= 1
|
||||
expect(metric_value('turniere_action_cable_connections_total')).to be >= 1
|
||||
ensure
|
||||
cable_client&.close
|
||||
end
|
||||
|
||||
def create_playoff_tournament
|
||||
owner_client = login_client
|
||||
unique = SecureRandom.hex(3)
|
||||
response = owner_client.post('/tournaments', body: {
|
||||
name: "Metrics #{unique}",
|
||||
public: false,
|
||||
teams: Array.new(4) { |index| { name: "Metrics-#{unique}-#{index + 1}" } }
|
||||
})
|
||||
|
||||
expect(response[:status]).to eq(201)
|
||||
response.fetch(:json)
|
||||
end
|
||||
|
||||
def login_client
|
||||
client = TurniereE2E::ApiClient.new(base_url:)
|
||||
response = client.login!(email: owner_email, password: owner_password)
|
||||
|
||||
expect(response[:status]).to eq(200)
|
||||
client
|
||||
end
|
||||
|
||||
def metric_value(metric_name)
|
||||
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 10
|
||||
loop do
|
||||
value = latest_metrics_lines
|
||||
.grep(/\A#{Regexp.escape(metric_name)}(?:\{| )/)
|
||||
.map { |line| line.split.last.to_f }
|
||||
.max
|
||||
return value unless value.nil?
|
||||
|
||||
raise "#{metric_name} not reported" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
||||
|
||||
sleep 0.2
|
||||
end
|
||||
end
|
||||
|
||||
def latest_metrics_lines
|
||||
response = anonymous_client.get('/metrics')
|
||||
expect(response[:status]).to eq(200)
|
||||
raw_body(response).lines.map(&:strip)
|
||||
end
|
||||
|
||||
def raw_body(response)
|
||||
response.dig(:json, :raw_body).to_s
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe 'Metrics endpoint', type: :request do
|
||||
before do
|
||||
MetricsRegistry.reset!
|
||||
end
|
||||
|
||||
it 'renders Prometheus text without authentication' do
|
||||
get '/metrics'
|
||||
|
||||
expect(response).to have_http_status(:ok)
|
||||
expect(response.media_type).to include('text/plain')
|
||||
expect(response.body).to include('# TYPE turniere_build_info gauge')
|
||||
expect(response.body).to include('# TYPE turniere_http_requests_total counter')
|
||||
expect(response.body).to include('turniere_action_cable_connections 0')
|
||||
end
|
||||
|
||||
it 'includes request metrics from previous controller requests' do
|
||||
get '/healthz'
|
||||
get '/metrics'
|
||||
|
||||
expect(response.body).to include(
|
||||
'turniere_http_requests_total{method="GET",controller="health",action="index",status="200"} 1'
|
||||
)
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe MetricsController, type: :routing do
|
||||
describe 'routing' do
|
||||
it 'routes to #index' do
|
||||
expect(get: '/metrics').to route_to('metrics#index')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe MetricsRegistry do
|
||||
let(:health_labels) { 'method="GET",controller="health",action="index",status="200"' }
|
||||
|
||||
before do
|
||||
described_class.reset!
|
||||
end
|
||||
|
||||
describe '.record_http_request' do
|
||||
it 'renders Prometheus counters and duration histogram buckets' do
|
||||
described_class.record_http_request(
|
||||
method: 'get',
|
||||
controller: 'health',
|
||||
action: 'index',
|
||||
status: 200,
|
||||
duration: 0.012
|
||||
)
|
||||
|
||||
metrics = described_class.render_prometheus
|
||||
|
||||
expect(metrics).to include(
|
||||
"turniere_http_requests_total{#{health_labels}} 1"
|
||||
)
|
||||
expect(metrics).to include(
|
||||
"turniere_http_request_duration_seconds_bucket{#{health_labels},le=\"0.025\"} 1"
|
||||
)
|
||||
expect(metrics).to include(
|
||||
"turniere_http_request_duration_seconds_count{#{health_labels}} 1"
|
||||
)
|
||||
expect(metrics).to match(
|
||||
/turniere_http_request_duration_seconds_sum\{#{health_labels}\} 0\.012/
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
describe 'ActionCable connection metrics' do
|
||||
it 'tracks open and total websocket connections' do
|
||||
described_class.action_cable_connected
|
||||
described_class.action_cable_connected
|
||||
described_class.action_cable_disconnected
|
||||
|
||||
metrics = described_class.render_prometheus
|
||||
|
||||
expect(metrics).to include("turniere_action_cable_connections 1\n")
|
||||
expect(metrics).to include("turniere_action_cable_connections_total 2\n")
|
||||
end
|
||||
|
||||
it 'does not let open websocket connections become negative' do
|
||||
described_class.action_cable_disconnected
|
||||
|
||||
metrics = described_class.render_prometheus
|
||||
|
||||
expect(metrics).to include("turniere_action_cable_connections 0\n")
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue