futu_server/

metrics.rs

// 网关运行时监控指标
//
// 使用原子计数器，零开销采集，无需外部依赖。
// 通过 telnet `show_metrics` 命令查看。
// v1.4.90 P1-B: 也通过 [`GatewayMetrics::render_prometheus`] 暴露到
// `/metrics` HTTP 端点 (经 [`futu_auth::metrics::Registry`] extension renderer
// 注册). 之前 v1.4.83/84 声称这些 counter 在 Prometheus 但仅在 telnet 输出.

use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;

use chrono::{Timelike, Utc};
use parking_lot::RwLock;

/// v1.4.84 §14: per-cmd_id per-UTC-hour breakdown for monitoring.
///
/// 让 tester CI 长窗口 job 能做时段异常检测 (CMD14716 UTC 15-18 window).
/// 单独结构, 各 cmd counter 独立持有一组 24 bucket.
#[derive(Debug)]
pub struct HourBreakdown {
    counters: [AtomicU64; 24],
}

impl HourBreakdown {
    pub const fn new() -> Self {
        Self {
            counters: [const { AtomicU64::new(0) }; 24],
        }
    }

    /// Bump the counter for current UTC hour (0..23).
    pub fn bump_now(&self) {
        let hour = Utc::now().hour() as usize;
        if hour < 24 {
            self.counters[hour].fetch_add(1, Ordering::Relaxed);
        }
    }

    /// Read counter for specific hour (0..23). Out-of-range returns 0.
    pub fn get(&self, hour: usize) -> u64 {
        self.counters
            .get(hour)
            .map(|a| a.load(Ordering::Relaxed))
            .unwrap_or(0)
    }

    /// Return snapshot of all 24 hours as array.
    pub fn snapshot(&self) -> [u64; 24] {
        let mut out = [0u64; 24];
        for (i, c) in self.counters.iter().enumerate() {
            out[i] = c.load(Ordering::Relaxed);
        }
        out
    }
}

impl Default for HourBreakdown {
    fn default() -> Self {
        Self::new()
    }
}

/// 网关运行时监控指标
pub struct GatewayMetrics {
    /// 网关启动时间
    pub start_time: Instant,

    // ===== 连接指标 =====
    /// 累计接受的客户端连接数
    pub total_connections: AtomicU64,
    /// 累计客户端断开数
    pub total_disconnections: AtomicU64,
    /// 被拒绝的连接数（超过上限）
    pub rejected_connections: AtomicU64,

    // ===== 请求指标 =====
    /// 累计处理的请求数
    pub total_requests: AtomicU64,
    /// 累计请求错误数（handler 返回 None 或解密失败）
    pub total_request_errors: AtomicU64,
    /// 累计响应字节数
    pub total_response_bytes: AtomicU64,

    // ===== 后端指标 =====
    /// 后端重连次数
    pub backend_reconnects: AtomicU64,
    /// 后端重连失败次数
    pub backend_reconnect_failures: AtomicU64,
    /// 最近一次重连时间 (Unix 毫秒, 0=未重连过)
    pub last_reconnect_ms: AtomicU64,
    /// 后端是否在线 (1=online, 0=offline)
    pub backend_online: AtomicU64,

    // ===== 推送指标 =====
    /// 后端收到的推送数 (CMD 6212 / 4716 / 5300 等)
    pub backend_pushes_received: AtomicU64,
    /// 向客户端发送的推送数
    pub client_pushes_sent: AtomicU64,
    // v1.4.83 §14 Phase 4: per-cmd_id push 细分计数 (monitoring)
    /// CMD 6212 quote push 计数
    pub backend_pushes_cmd_quote: AtomicU64,
    /// CMD 4716 trade notify (legacy channel) 计数
    pub backend_pushes_cmd_trade_legacy: AtomicU64,
    /// CMD 14716 trade notify (v1.4.41 new channel) 计数 — tester §14 追踪
    pub backend_pushes_cmd_trade_new: AtomicU64,
    /// CMD 5300 msg-center push 计数
    pub backend_pushes_cmd_msg_center: AtomicU64,
    /// 其他未路由 CMD push 计数
    pub backend_pushes_cmd_other: AtomicU64,

    // v1.4.84 §14: per-cmd_id × UTC-hour 分桶 (监控深化 — tester CI 长窗口
    // job 用于 "CMD14716 UTC 15-18 异常时段" 检测). cmd_other 不分时段.
    /// CMD 6212 quote push per-UTC-hour 计数
    pub backend_pushes_cmd_quote_by_hour: HourBreakdown,
    /// CMD 4716 trade notify (legacy) per-UTC-hour 计数
    pub backend_pushes_cmd_trade_legacy_by_hour: HourBreakdown,
    /// CMD 14716 trade notify (new) per-UTC-hour 计数 — tester §14 主角
    pub backend_pushes_cmd_trade_new_by_hour: HourBreakdown,
    /// CMD 5300 msg-center push per-UTC-hour 计数
    pub backend_pushes_cmd_msg_center_by_hour: HourBreakdown,

    // ===== 订阅指标 =====
    /// 行情订阅操作次数
    pub qot_subscribe_ops: AtomicU64,
    /// 行情退订操作次数
    pub qot_unsubscribe_ops: AtomicU64,

    // ===== v1.4.110 codex audit Round2 P3 #19: cold-cache wait 监控 =====
    //
    // GetBasicQot / GetOrderBook cache miss + 已订阅 → cold-cache wait 路径
    // (Pull_SubData 主动拉 + 最多 3s 等 push). ops 用 hit/total 比看 backend
    // push 延迟健康度, timeout/total 比看 cold-cache wait 是否常超时.
    /// cold-cache wait 进入次数 (cache miss + IsSub, 触发 wait)
    pub cold_cache_wait_total: AtomicU64,
    /// cold-cache wait 命中次数 (3s 内 push 写 cache → re-read 拿到值)
    pub cold_cache_wait_hit: AtomicU64,
    /// cold-cache wait 超时次数 (3s timeout 仍 cache miss)
    pub cold_cache_wait_timeout: AtomicU64,
    /// 重连后重新订阅次数 (legacy, == applied_keys 累加; v1.4.106 codex 0631
    /// F5 起仍 bump 向后兼容旧 dashboard, 等价新 `resubscribe_applied_keys`).
    pub resubscribe_ops: AtomicU64,

    // ===== v1.4.106 codex 0631 F5 [P3]: dual resubscribe counter =====
    //
    // 老 `resubscribe_ops_total` 把"触发数"和"真生效 keys 数"混一桶, 看不出
    // partial / cache miss 的 silent loss. 拆 dual:
    //   - resubscribe_attempts_total: 触发次数 (本 reconnect / staleness 触
    //     发了 N 次 resubscribe, 不论结果).
    //   - resubscribe_applied_keys_total: 真生效 keys 数 (cache resolve OK +
    //     backend ack OK 的 (sec_key, sub_type) 对). 部分失败时 < attempts.
    //
    // ratio applied/attempts < 1.0 显著 → ghost subs / cache miss / backend
    // partial reject 信号. legacy resubscribe_ops_total 仍 bump (向后兼容).
    /// **v1.4.106 codex 0631 F5 [P3]**: resubscribe 触发次数 (每次 reconnect /
    /// staleness loop 触发 += 1). 与 applied_keys 对比 ratio 看 partial 程度.
    pub resubscribe_attempts: AtomicU64,
    /// **v1.4.106 codex 0631 F5 [P3]**: resubscribe 真生效 keys 数 (累积).
    /// applied < attempts × global_keys → 部分 partial.
    pub resubscribe_applied_keys: AtomicU64,

    // ===== v1.4.106 codex 1140 F8: 行情 push 投递失败计数 =====
    //
    // 之前 `bridge::push_parser` 用 `let _ = push_tx.try_send(event)` 静默吞错,
    // cache 已更新但 subscriber 收不到 push (audit Finding 8). 加 metric +
    // warn log 让 channel full / closed 立即可观测.
    /// CMD 6212 行情 push (BasicQot/OrderBook/Ticker/RT/KL/Broker/...) 因
    /// `push_tx` 队列满或关闭被 drop 的总次数 (累积).
    pub qot_push_dropped_total: AtomicU64,
    /// 按 SubType 拆分的 drop 计数 (proto Qot_Common.SubType: 0..=17, 共 18 桶).
    /// 桶 0 = "未知 / 不属于任何已知 SubType" (兜底, e.g. 拼错的 sub_type).
    /// 桶 1..=17 = 对应 SubType. v1.4.106 codex 1140 F8 加.
    pub qot_push_dropped_by_sub_type: [AtomicU64; 18],

    // ===== KeepAlive 指标 =====
    /// KeepAlive 超时断开数
    pub keepalive_timeouts: AtomicU64,

    // ===== 延迟采样 =====
    /// 最近 N 个请求延迟的环形缓冲 (纳秒)
    latency_ring: RwLock<LatencyRing>,
}

/// 延迟环形缓冲 — 保留最近 1000 个采样
struct LatencyRing {
    buf: Vec<u64>,
    pos: usize,
    count: u64,
    total_ns: u64,
}

const LATENCY_RING_SIZE: usize = 1000;

impl LatencyRing {
    fn new() -> Self {
        Self {
            buf: vec![0u64; LATENCY_RING_SIZE],
            pos: 0,
            count: 0,
            total_ns: 0,
        }
    }

    fn push(&mut self, ns: u64) {
        // 减去被覆盖的旧值
        if self.count >= LATENCY_RING_SIZE as u64 {
            self.total_ns = self.total_ns.saturating_sub(self.buf[self.pos]);
        }
        self.buf[self.pos] = ns;
        self.total_ns += ns;
        self.pos = (self.pos + 1) % LATENCY_RING_SIZE;
        self.count += 1;
    }

    fn stats(&self) -> LatencyStats {
        let n = self.count.min(LATENCY_RING_SIZE as u64) as usize;
        if n == 0 {
            return LatencyStats::default();
        }

        let mut samples: Vec<u64> = if self.count >= LATENCY_RING_SIZE as u64 {
            self.buf.clone()
        } else {
            self.buf[..n].to_vec()
        };
        samples.sort_unstable();

        LatencyStats {
            count: self.count,
            avg_us: (self.total_ns / n as u64) / 1000,
            p50_us: samples[n / 2] / 1000,
            p95_us: samples[(n as f64 * 0.95) as usize] / 1000,
            p99_us: samples[(n as f64 * 0.99).min((n - 1) as f64) as usize] / 1000,
            max_us: samples[n - 1] / 1000,
        }
    }
}

/// 延迟统计摘要 (微秒)
#[derive(Default)]
pub struct LatencyStats {
    /// 总采样数
    pub count: u64,
    /// 平均延迟 (微秒)
    pub avg_us: u64,
    /// P50 延迟
    pub p50_us: u64,
    /// P95 延迟
    pub p95_us: u64,
    /// P99 延迟
    pub p99_us: u64,
    /// 最大延迟
    pub max_us: u64,
}

/// v1.4.84 §14: 把 24 小时 counters snapshot 格式化为空格分隔的单行.
///
/// 输出格式: `"h00=N h01=N ... h23=N"` — 便于人眼 scan 时段异常,
/// 同时保持 parseable (awk / grep / Prometheus textfile).
fn format_hour_row(hb: &HourBreakdown) -> String {
    let snap = hb.snapshot();
    let mut out = String::with_capacity(24 * 10);
    for (i, v) in snap.iter().enumerate() {
        if i > 0 {
            out.push(' ');
        }
        out.push_str(&format!("h{:02}={}", i, v));
    }
    out
}

impl GatewayMetrics {
    pub fn new() -> Self {
        Self {
            start_time: Instant::now(),
            total_connections: AtomicU64::new(0),
            total_disconnections: AtomicU64::new(0),
            rejected_connections: AtomicU64::new(0),
            total_requests: AtomicU64::new(0),
            total_request_errors: AtomicU64::new(0),
            total_response_bytes: AtomicU64::new(0),
            backend_reconnects: AtomicU64::new(0),
            backend_reconnect_failures: AtomicU64::new(0),
            last_reconnect_ms: AtomicU64::new(0),
            backend_online: AtomicU64::new(1),
            backend_pushes_received: AtomicU64::new(0),
            client_pushes_sent: AtomicU64::new(0),
            backend_pushes_cmd_quote: AtomicU64::new(0),
            backend_pushes_cmd_trade_legacy: AtomicU64::new(0),
            backend_pushes_cmd_trade_new: AtomicU64::new(0),
            backend_pushes_cmd_msg_center: AtomicU64::new(0),
            backend_pushes_cmd_other: AtomicU64::new(0),
            backend_pushes_cmd_quote_by_hour: HourBreakdown::new(),
            backend_pushes_cmd_trade_legacy_by_hour: HourBreakdown::new(),
            backend_pushes_cmd_trade_new_by_hour: HourBreakdown::new(),
            backend_pushes_cmd_msg_center_by_hour: HourBreakdown::new(),
            qot_subscribe_ops: AtomicU64::new(0),
            qot_unsubscribe_ops: AtomicU64::new(0),
            cold_cache_wait_total: AtomicU64::new(0),
            cold_cache_wait_hit: AtomicU64::new(0),
            cold_cache_wait_timeout: AtomicU64::new(0),
            resubscribe_ops: AtomicU64::new(0),
            resubscribe_attempts: AtomicU64::new(0),
            resubscribe_applied_keys: AtomicU64::new(0),
            // v1.4.106 codex 1140 F8: qot push drop counter init.
            qot_push_dropped_total: AtomicU64::new(0),
            qot_push_dropped_by_sub_type: [const { AtomicU64::new(0) }; 18],
            keepalive_timeouts: AtomicU64::new(0),
            latency_ring: RwLock::new(LatencyRing::new()),
        }
    }

    /// 记录一次请求延迟 (纳秒)
    pub fn record_latency_ns(&self, ns: u64) {
        self.latency_ring.write().push(ns);
    }

    /// v1.4.106 codex 1140 F8: 记录一次 qot push 被 drop (channel full / closed).
    ///
    /// `sub_type` 范围 0..=17 (proto Qot_Common.SubType). 越界值归桶 0
    /// (未知). 同时 bump 总计数 + per-sub-type 桶, 保证 dashboard 可分维度.
    pub fn record_qot_push_dropped(&self, sub_type: i32) {
        self.qot_push_dropped_total.fetch_add(1, Ordering::Relaxed);
        let bucket = if (0..18).contains(&sub_type) {
            sub_type as usize
        } else {
            0
        };
        self.qot_push_dropped_by_sub_type[bucket].fetch_add(1, Ordering::Relaxed);
    }

    /// v1.4.106 codex 1140 F8: 读取每个 sub_type 桶的 drop 计数 (snapshot).
    /// 用于 metrics endpoint render.
    pub fn qot_push_dropped_per_sub_type(&self) -> [u64; 18] {
        let mut out = [0u64; 18];
        for (i, slot) in self.qot_push_dropped_by_sub_type.iter().enumerate() {
            out[i] = slot.load(Ordering::Relaxed);
        }
        out
    }

    /// 获取延迟统计
    pub fn latency_stats(&self) -> LatencyStats {
        self.latency_ring.read().stats()
    }

    /// 格式化运行时间
    pub fn uptime_str(&self) -> String {
        let elapsed = self.start_time.elapsed();
        let secs = elapsed.as_secs();
        let days = secs / 86400;
        let hours = (secs % 86400) / 3600;
        let mins = (secs % 3600) / 60;
        let s = secs % 60;
        if days > 0 {
            format!("{days}d {hours}h {mins}m {s}s")
        } else if hours > 0 {
            format!("{hours}h {mins}m {s}s")
        } else {
            format!("{mins}m {s}s")
        }
    }

    /// 生成 telnet 可展示的指标报告
    pub fn report(&self) -> String {
        let lat = self.latency_stats();
        let backend_status = if self.backend_online.load(Ordering::Relaxed) == 1 {
            "ONLINE"
        } else {
            "OFFLINE"
        };

        let total_req = self.total_requests.load(Ordering::Relaxed);
        let uptime_secs = self.start_time.elapsed().as_secs_f64();
        let avg_rps = if uptime_secs > 0.0 {
            total_req as f64 / uptime_secs
        } else {
            0.0
        };

        format!(
            "=== Gateway Metrics ===\r\n\
             Uptime: {uptime}\r\n\
             \r\n\
             [Connections]\r\n\
             total_accepted: {total_conn}\r\n\
             total_disconnected: {total_disconn}\r\n\
             rejected (limit): {rejected}\r\n\
             keepalive_timeouts: {ka_timeout}\r\n\
             \r\n\
             [Requests]\r\n\
             total_requests: {total_req}\r\n\
             total_errors: {total_err}\r\n\
             avg_rps: {avg_rps:.1}\r\n\
             response_bytes: {resp_bytes}\r\n\
             \r\n\
             [Latency (recent {lat_count} samples)]\r\n\
             avg: {lat_avg}us  p50: {lat_p50}us  p95: {lat_p95}us  p99: {lat_p99}us  max: {lat_max}us\r\n\
             \r\n\
             [Backend]\r\n\
             status: {backend_status}\r\n\
             reconnects: {reconnects}\r\n\
             reconnect_failures: {reconnect_fail}\r\n\
             pushes_received: {push_recv}\r\n\
             pushes_sent_to_clients: {push_sent}\r\n\
             \r\n\
             [Pushes by CMD (v1.4.83 §14)]\r\n\
             cmd_6212_quote: {push_cmd_quote}\r\n\
             cmd_4716_trade_legacy: {push_cmd_trade_legacy}\r\n\
             cmd_14716_trade_new: {push_cmd_trade_new}\r\n\
             cmd_5300_msg_center: {push_cmd_msg_center}\r\n\
             cmd_other: {push_cmd_other}\r\n\
             \r\n\
             [Pushes by CMD × UTC hour (v1.4.84 §14)]\r\n\
             cmd_14716_trade_new_hour_0..23: {hour_trade_new}\r\n\
             cmd_6212_quote_hour_0..23: {hour_quote}\r\n\
             cmd_4716_trade_legacy_hour_0..23: {hour_trade_legacy}\r\n\
             cmd_5300_msg_center_hour_0..23: {hour_msg_center}\r\n\
             \r\n\
             [Subscriptions]\r\n\
             subscribe_ops: {sub_ops}\r\n\
             unsubscribe_ops: {unsub_ops}\r\n\
             resubscribe_ops: {resub_ops}\r\n\
             \r\n\
             [Cold-cache wait (v1.4.110 §P3 #19)]\r\n\
             total: {cc_total}  hit: {cc_hit}  timeout: {cc_timeout}\r\n",
            uptime = self.uptime_str(),
            total_conn = self.total_connections.load(Ordering::Relaxed),
            total_disconn = self.total_disconnections.load(Ordering::Relaxed),
            rejected = self.rejected_connections.load(Ordering::Relaxed),
            ka_timeout = self.keepalive_timeouts.load(Ordering::Relaxed),
            total_req = total_req,
            total_err = self.total_request_errors.load(Ordering::Relaxed),
            resp_bytes = self.total_response_bytes.load(Ordering::Relaxed),
            lat_count = lat.count.min(LATENCY_RING_SIZE as u64),
            lat_avg = lat.avg_us,
            lat_p50 = lat.p50_us,
            lat_p95 = lat.p95_us,
            lat_p99 = lat.p99_us,
            lat_max = lat.max_us,
            reconnects = self.backend_reconnects.load(Ordering::Relaxed),
            reconnect_fail = self.backend_reconnect_failures.load(Ordering::Relaxed),
            push_recv = self.backend_pushes_received.load(Ordering::Relaxed),
            push_sent = self.client_pushes_sent.load(Ordering::Relaxed),
            push_cmd_quote = self.backend_pushes_cmd_quote.load(Ordering::Relaxed),
            push_cmd_trade_legacy = self.backend_pushes_cmd_trade_legacy.load(Ordering::Relaxed),
            push_cmd_trade_new = self.backend_pushes_cmd_trade_new.load(Ordering::Relaxed),
            push_cmd_msg_center = self.backend_pushes_cmd_msg_center.load(Ordering::Relaxed),
            push_cmd_other = self.backend_pushes_cmd_other.load(Ordering::Relaxed),
            hour_trade_new = format_hour_row(&self.backend_pushes_cmd_trade_new_by_hour),
            hour_quote = format_hour_row(&self.backend_pushes_cmd_quote_by_hour),
            hour_trade_legacy = format_hour_row(&self.backend_pushes_cmd_trade_legacy_by_hour),
            hour_msg_center = format_hour_row(&self.backend_pushes_cmd_msg_center_by_hour),
            sub_ops = self.qot_subscribe_ops.load(Ordering::Relaxed),
            unsub_ops = self.qot_unsubscribe_ops.load(Ordering::Relaxed),
            resub_ops = self.resubscribe_ops.load(Ordering::Relaxed),
            cc_total = self.cold_cache_wait_total.load(Ordering::Relaxed),
            cc_hit = self.cold_cache_wait_hit.load(Ordering::Relaxed),
            cc_timeout = self.cold_cache_wait_timeout.load(Ordering::Relaxed),
        )
    }
}

impl Default for GatewayMetrics {
    fn default() -> Self {
        Self::new()
    }
}

/// v1.4.90 P1-B: 把 24 小时 hour breakdown 渲染成 Prometheus 行 (with `hour` label).
///
/// 输出形如 `<metric_name>{hour="00"} 7\n<metric_name>{hour="01"} 0\n...`
/// 每个 metric 24 行 (UTC hour 0..23).
fn render_hour_breakdown_prom(metric_name: &str, hb: &HourBreakdown) -> String {
    let snap = hb.snapshot();
    let mut out = String::with_capacity(24 * 60);
    for (h, v) in snap.iter().enumerate() {
        out.push_str(&format!("{}{{hour=\"{:02}\"}} {}\n", metric_name, h, v));
    }
    out
}

impl GatewayMetrics {
    /// v1.4.90 P1-B: 输出 Prometheus text exposition 格式, 涵盖:
    ///
    /// - 连接 / 请求 / 响应 / 后端连接基础 counter
    /// - per-cmd_id push counter (cmd_6212_quote / cmd_4716 / cmd_14716 /
    ///   cmd_5300 / cmd_other) — v1.4.83/84 telnet 已暴露, 此处补 Prometheus
    /// - per-cmd_id × UTC-hour 24-bucket breakdown (v1.4.84 §14)
    /// - 订阅 / KeepAlive 计数
    /// - 延迟 p50/p95/p99 gauge (取最近 1000 样本)
    ///
    /// 注册到 [`futu_auth::metrics::Registry`] extension renderer 后, 在
    /// `/metrics` HTTP 输出末尾自动追加. 见
    /// [`install_prometheus_extension`].
    #[must_use]
    pub fn render_prometheus(&self) -> String {
        let mut s = String::with_capacity(8192);

        // ===== 连接 =====
        s.push_str("# HELP futu_gateway_connections_total Total accepted client connections\n");
        s.push_str("# TYPE futu_gateway_connections_total counter\n");
        s.push_str(&format!(
            "futu_gateway_connections_total {}\n",
            self.total_connections.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_disconnections_total Total client disconnections\n# TYPE futu_gateway_disconnections_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_disconnections_total {}\n",
            self.total_disconnections.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_rejected_connections_total Connections rejected (limit hit)\n# TYPE futu_gateway_rejected_connections_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_rejected_connections_total {}\n",
            self.rejected_connections.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_keepalive_timeouts_total KeepAlive timeout disconnects\n# TYPE futu_gateway_keepalive_timeouts_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_keepalive_timeouts_total {}\n",
            self.keepalive_timeouts.load(Ordering::Relaxed)
        ));

        // ===== 请求 =====
        s.push_str(
            "# HELP futu_gateway_requests_total Total handled client requests\n# TYPE futu_gateway_requests_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_requests_total {}\n",
            self.total_requests.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_request_errors_total Handler-returned-None or decryption errors\n# TYPE futu_gateway_request_errors_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_request_errors_total {}\n",
            self.total_request_errors.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_response_bytes_total Cumulative response payload bytes\n# TYPE futu_gateway_response_bytes_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_response_bytes_total {}\n",
            self.total_response_bytes.load(Ordering::Relaxed)
        ));

        // ===== 后端 =====
        s.push_str(
            "# HELP futu_gateway_backend_online Backend connection state (1=online,0=offline)\n# TYPE futu_gateway_backend_online gauge\n",
        );
        s.push_str(&format!(
            "futu_gateway_backend_online {}\n",
            self.backend_online.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_backend_reconnects_total Backend reconnect attempts\n# TYPE futu_gateway_backend_reconnects_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_backend_reconnects_total {}\n",
            self.backend_reconnects.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_backend_reconnect_failures_total Backend reconnect failures\n# TYPE futu_gateway_backend_reconnect_failures_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_backend_reconnect_failures_total {}\n",
            self.backend_reconnect_failures.load(Ordering::Relaxed)
        ));

        // ===== 推送总数 =====
        s.push_str(
            "# HELP futu_gateway_backend_pushes_received_total Pushes received from backend\n# TYPE futu_gateway_backend_pushes_received_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_backend_pushes_received_total {}\n",
            self.backend_pushes_received.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_client_pushes_sent_total Pushes forwarded to clients\n# TYPE futu_gateway_client_pushes_sent_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_client_pushes_sent_total {}\n",
            self.client_pushes_sent.load(Ordering::Relaxed)
        ));

        // ===== per-cmd push counters (v1.4.83 §14) =====
        // 这是 P1-B 修复的核心: telnet 已有但 /metrics 之前没暴露
        s.push_str(
            "# HELP futu_gateway_backend_pushes_cmd_total Backend pushes by cmd_id (v1.4.83 §14)\n# TYPE futu_gateway_backend_pushes_cmd_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_backend_pushes_cmd_total{{cmd=\"6212_quote\"}} {}\n",
            self.backend_pushes_cmd_quote.load(Ordering::Relaxed)
        ));
        s.push_str(&format!(
            "futu_gateway_backend_pushes_cmd_total{{cmd=\"4716_trade_legacy\"}} {}\n",
            self.backend_pushes_cmd_trade_legacy.load(Ordering::Relaxed)
        ));
        s.push_str(&format!(
            "futu_gateway_backend_pushes_cmd_total{{cmd=\"14716_trade_new\"}} {}\n",
            self.backend_pushes_cmd_trade_new.load(Ordering::Relaxed)
        ));
        s.push_str(&format!(
            "futu_gateway_backend_pushes_cmd_total{{cmd=\"5300_msg_center\"}} {}\n",
            self.backend_pushes_cmd_msg_center.load(Ordering::Relaxed)
        ));
        s.push_str(&format!(
            "futu_gateway_backend_pushes_cmd_total{{cmd=\"other\"}} {}\n",
            self.backend_pushes_cmd_other.load(Ordering::Relaxed)
        ));

        // ===== per-cmd × UTC hour breakdown (v1.4.84 §14) =====
        s.push_str(
            "# HELP futu_gateway_backend_pushes_cmd_quote_by_hour Cmd 6212 quote pushes per UTC hour\n# TYPE futu_gateway_backend_pushes_cmd_quote_by_hour counter\n",
        );
        s.push_str(&render_hour_breakdown_prom(
            "futu_gateway_backend_pushes_cmd_quote_by_hour",
            &self.backend_pushes_cmd_quote_by_hour,
        ));
        s.push_str(
            "# HELP futu_gateway_backend_pushes_cmd_trade_legacy_by_hour Cmd 4716 trade-legacy pushes per UTC hour\n# TYPE futu_gateway_backend_pushes_cmd_trade_legacy_by_hour counter\n",
        );
        s.push_str(&render_hour_breakdown_prom(
            "futu_gateway_backend_pushes_cmd_trade_legacy_by_hour",
            &self.backend_pushes_cmd_trade_legacy_by_hour,
        ));
        s.push_str(
            "# HELP futu_gateway_backend_pushes_cmd_trade_new_by_hour Cmd 14716 trade-new pushes per UTC hour (v1.4.84 §14 tester subject)\n# TYPE futu_gateway_backend_pushes_cmd_trade_new_by_hour counter\n",
        );
        s.push_str(&render_hour_breakdown_prom(
            "futu_gateway_backend_pushes_cmd_trade_new_by_hour",
            &self.backend_pushes_cmd_trade_new_by_hour,
        ));
        s.push_str(
            "# HELP futu_gateway_backend_pushes_cmd_msg_center_by_hour Cmd 5300 msg-center pushes per UTC hour\n# TYPE futu_gateway_backend_pushes_cmd_msg_center_by_hour counter\n",
        );
        s.push_str(&render_hour_breakdown_prom(
            "futu_gateway_backend_pushes_cmd_msg_center_by_hour",
            &self.backend_pushes_cmd_msg_center_by_hour,
        ));

        // ===== 订阅 =====
        s.push_str(
            "# HELP futu_gateway_qot_subscribe_ops_total Quote subscribe operations\n# TYPE futu_gateway_qot_subscribe_ops_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_qot_subscribe_ops_total {}\n",
            self.qot_subscribe_ops.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_qot_unsubscribe_ops_total Quote unsubscribe operations\n# TYPE futu_gateway_qot_unsubscribe_ops_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_qot_unsubscribe_ops_total {}\n",
            self.qot_unsubscribe_ops.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_resubscribe_ops_total Re-subscribe ops after reconnect (legacy, == resubscribe_applied_keys)\n# TYPE futu_gateway_resubscribe_ops_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_resubscribe_ops_total {}\n",
            self.resubscribe_ops.load(Ordering::Relaxed)
        ));
        // v1.4.106 codex 0631 F5: dual counter — attempts 触发 vs applied_keys 真生效.
        s.push_str(
            "# HELP futu_gateway_resubscribe_attempts_total Re-subscribe trigger count (each reconnect/staleness loop +=1)\n# TYPE futu_gateway_resubscribe_attempts_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_resubscribe_attempts_total {}\n",
            self.resubscribe_attempts.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_resubscribe_applied_keys_total Re-subscribe applied keys total (cache resolve OK + backend ack OK)\n# TYPE futu_gateway_resubscribe_applied_keys_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_resubscribe_applied_keys_total {}\n",
            self.resubscribe_applied_keys.load(Ordering::Relaxed)
        ));

        // v1.4.110 codex audit Round2 P3 #19: cold-cache wait 监控.
        // hit/total 比看 backend push 延迟健康度, timeout/total 比看常超时.
        s.push_str(
            "# HELP futu_gateway_cold_cache_wait_total Cold-cache wait entries (cache miss + IsSub)\n# TYPE futu_gateway_cold_cache_wait_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_cold_cache_wait_total {}\n",
            self.cold_cache_wait_total.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_cold_cache_wait_hit_total Cold-cache wait hits (push filled cache within timeout)\n# TYPE futu_gateway_cold_cache_wait_hit_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_cold_cache_wait_hit_total {}\n",
            self.cold_cache_wait_hit.load(Ordering::Relaxed)
        ));
        s.push_str(
            "# HELP futu_gateway_cold_cache_wait_timeout_total Cold-cache wait timeouts (3s elapsed, cache still miss)\n# TYPE futu_gateway_cold_cache_wait_timeout_total counter\n",
        );
        s.push_str(&format!(
            "futu_gateway_cold_cache_wait_timeout_total {}\n",
            self.cold_cache_wait_timeout.load(Ordering::Relaxed)
        ));

        // ===== 延迟 (gauge, 取 ring 当前 stats) =====
        let lat = self.latency_stats();
        s.push_str(
            "# HELP futu_gateway_request_latency_us Request latency percentiles (microseconds, recent ring)\n# TYPE futu_gateway_request_latency_us gauge\n",
        );
        s.push_str(&format!(
            "futu_gateway_request_latency_us{{quantile=\"p50\"}} {}\n",
            lat.p50_us
        ));
        s.push_str(&format!(
            "futu_gateway_request_latency_us{{quantile=\"p95\"}} {}\n",
            lat.p95_us
        ));
        s.push_str(&format!(
            "futu_gateway_request_latency_us{{quantile=\"p99\"}} {}\n",
            lat.p99_us
        ));
        s.push_str(&format!(
            "futu_gateway_request_latency_us{{quantile=\"max\"}} {}\n",
            lat.max_us
        ));

        s
    }
}

/// v1.4.90 P1-B: 把 [`GatewayMetrics`] 注册为 [`futu_auth::metrics::Registry`]
/// 的 extension renderer, 让 `/metrics` HTTP 端点自动包含 per-cmd / per-hour
/// counter.
///
/// 调用方 (futu-opend `main.rs`) 在创建 `GatewayMetrics` Arc 之后调一次:
///
/// ```ignore
/// futu_auth::metrics::install(Arc::new(MetricsRegistry::default()));
/// // ... bridge / server 初始化, 都共享同一份 Arc<GatewayMetrics> ...
/// futu_server::metrics::install_prometheus_extension(server.metrics().clone());
/// ```
///
/// 多次调用会注册多个 renderer (无害但重复输出); 实际上只该调用一次.
pub fn install_prometheus_extension(metrics: Arc<GatewayMetrics>) {
    futu_auth::metrics::register_global_renderer(move || metrics.render_prometheus());
}

#[cfg(test)]
mod tests;