futu_server/

metrics.rs

// 网关运行时监控指标
//
// 使用原子计数器，零开销采集，无需外部依赖。
// 通过 telnet `show_metrics` 命令查看。
// v1.4.90 P1-B: 也通过 [`GatewayMetrics::render_prometheus`] 暴露到
// `/metrics` HTTP 端点 (经 [`futu_auth::metrics::Registry`] extension renderer
// 注册). 之前 v1.4.83/84 声称这些 counter 在 Prometheus 但仅在 telnet 输出.

use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;

use chrono::{Timelike, Utc};
use parking_lot::RwLock;

mod prometheus;
pub use prometheus::install_prometheus_extension;

/// v1.4.84 §14: per-cmd_id per-UTC-hour breakdown for monitoring.
///
/// 让 tester CI 长窗口 job 能做时段异常检测 (CMD14716 UTC 15-18 window).
/// 单独结构, 各 cmd counter 独立持有一组 24 bucket.
#[derive(Debug)]
pub struct HourBreakdown {
    counters: [AtomicU64; 24],
}

impl HourBreakdown {
    pub const fn new() -> Self {
        Self {
            counters: [const { AtomicU64::new(0) }; 24],
        }
    }

    /// Bump the counter for current UTC hour (0..23).
    pub fn bump_now(&self) {
        let hour = Utc::now().hour() as usize;
        if hour < 24 {
            self.counters[hour].fetch_add(1, Ordering::Relaxed);
        }
    }

    /// Read counter for specific hour (0..23). Out-of-range returns 0.
    pub fn get(&self, hour: usize) -> u64 {
        self.counters
            .get(hour)
            .map(|a| a.load(Ordering::Relaxed))
            .unwrap_or(0)
    }

    /// Return snapshot of all 24 hours as array.
    pub fn snapshot(&self) -> [u64; 24] {
        let mut out = [0u64; 24];
        for (i, c) in self.counters.iter().enumerate() {
            out[i] = c.load(Ordering::Relaxed);
        }
        out
    }
}

impl Default for HourBreakdown {
    fn default() -> Self {
        Self::new()
    }
}

/// 网关运行时监控指标
pub struct GatewayMetrics {
    /// 网关启动时间
    pub start_time: Instant,

    // ===== 连接指标 =====
    /// 累计接受的客户端连接数
    pub total_connections: AtomicU64,
    /// 累计客户端断开数
    pub total_disconnections: AtomicU64,
    /// 被拒绝的连接数（超过上限）
    pub rejected_connections: AtomicU64,

    // ===== 请求指标 =====
    /// 累计处理的请求数
    pub total_requests: AtomicU64,
    /// 累计请求错误数（handler 返回 None 或解密失败）
    pub total_request_errors: AtomicU64,
    /// 累计响应字节数
    pub total_response_bytes: AtomicU64,

    // ===== 后端指标 =====
    /// 后端重连次数
    pub backend_reconnects: AtomicU64,
    /// 后端重连失败次数
    pub backend_reconnect_failures: AtomicU64,
    /// 最近一次重连时间 (Unix 毫秒, 0=未重连过)
    pub last_reconnect_ms: AtomicU64,
    /// 后端是否在线 (1=online, 0=offline)
    pub backend_online: AtomicU64,

    // ===== 推送指标 =====
    /// 后端收到的推送数 (CMD 6212 / 4716 / 5300 等)
    pub backend_pushes_received: AtomicU64,
    /// 向客户端发送的推送数
    pub client_pushes_sent: AtomicU64,
    /// 向客户端推送时 client channel 已关闭导致发送失败的次数
    pub client_push_send_failures: AtomicU64,
    /// 行情推送 fanout 时 client channel 已满、仅丢该慢客户端本帧的次数。
    ///
    /// 注意: `qot_push_dropped_total` 统计 backend → dispatcher 队列 drop；
    /// 本 counter 统计 dispatcher → client channel backpressure，语义不同。
    pub qot_client_push_backpressure_drops: AtomicU64,
    /// 按 SubType 拆分的 dispatcher → client channel backpressure drop 计数。
    /// 桶 0 = 未知 / 首推未携带 SubType；桶 1..=17 = 对应 Qot_Common.SubType。
    pub qot_client_push_backpressure_drops_by_sub_type: [AtomicU64; 18],
    // v1.4.83 §14 Phase 4: per-cmd_id push 细分计数 (monitoring)
    /// CMD 6212 quote push 计数
    pub backend_pushes_cmd_quote: AtomicU64,
    /// CMD 4716 trade notify (legacy channel) 计数
    pub backend_pushes_cmd_trade_legacy: AtomicU64,
    /// CMD 14716 trade notify (v1.4.41 new channel) 计数 — tester §14 追踪
    pub backend_pushes_cmd_trade_new: AtomicU64,
    /// CMD 5300 msg-center push 计数
    pub backend_pushes_cmd_msg_center: AtomicU64,
    /// 其他未路由 CMD push 计数
    pub backend_pushes_cmd_other: AtomicU64,

    // v1.4.84 §14: per-cmd_id × UTC-hour 分桶 (监控深化 — tester CI 长窗口
    // job 用于 "CMD14716 UTC 15-18 异常时段" 检测). cmd_other 不分时段.
    /// CMD 6212 quote push per-UTC-hour 计数
    pub backend_pushes_cmd_quote_by_hour: HourBreakdown,
    /// CMD 4716 trade notify (legacy) per-UTC-hour 计数
    pub backend_pushes_cmd_trade_legacy_by_hour: HourBreakdown,
    /// CMD 14716 trade notify (new) per-UTC-hour 计数 — tester §14 主角
    pub backend_pushes_cmd_trade_new_by_hour: HourBreakdown,
    /// CMD 5300 msg-center push per-UTC-hour 计数
    pub backend_pushes_cmd_msg_center_by_hour: HourBreakdown,

    // ===== 订阅指标 =====
    /// 行情订阅操作次数
    pub qot_subscribe_ops: AtomicU64,
    /// 行情退订操作次数
    pub qot_unsubscribe_ops: AtomicU64,

    // ===== v1.4.110 codex audit Round2 P3 #19: cold-cache wait 监控 =====
    //
    // GetBasicQot / GetOrderBook cache miss + 已订阅 → cold-cache wait 路径
    // (Pull_SubData 主动拉 + 最多 3s 等 push). ops 用 hit/total 比看 backend
    // push 延迟健康度, timeout/total 比看 cold-cache wait 是否常超时.
    /// cold-cache wait 进入次数 (cache miss + IsSub, 触发 wait)
    pub cold_cache_wait_total: AtomicU64,
    /// cold-cache wait 命中次数 (3s 内 push 写 cache → re-read 拿到值)
    pub cold_cache_wait_hit: AtomicU64,
    /// cold-cache wait 超时次数 (3s timeout 仍 cache miss)
    pub cold_cache_wait_timeout: AtomicU64,
    /// 重连后重新订阅次数 (legacy, == applied_keys 累加; v1.4.106 codex 0631
    /// F5 起仍 bump 向后兼容旧 dashboard, 等价新 `resubscribe_applied_keys`).
    pub resubscribe_ops: AtomicU64,

    // ===== v1.4.106 codex 0631 F5 [P3]: dual resubscribe counter =====
    //
    // 老 `resubscribe_ops_total` 把"触发数"和"真生效 keys 数"混一桶, 看不出
    // partial / cache miss 的 silent loss. 拆 dual:
    //   - resubscribe_attempts_total: 触发次数 (本 reconnect / staleness 触
    //     发了 N 次 resubscribe, 不论结果).
    //   - resubscribe_applied_keys_total: 真生效 keys 数 (cache resolve OK +
    //     backend ack OK 的 (sec_key, sub_type) 对). 部分失败时 < attempts.
    //
    // ratio applied/attempts < 1.0 显著 → ghost subs / cache miss / backend
    // partial reject 信号. legacy resubscribe_ops_total 仍 bump (向后兼容).
    /// **v1.4.106 codex 0631 F5 [P3]**: resubscribe 触发次数 (每次 reconnect /
    /// staleness loop 触发 += 1). 与 applied_keys 对比 ratio 看 partial 程度.
    pub resubscribe_attempts: AtomicU64,
    /// **v1.4.106 codex 0631 F5 [P3]**: resubscribe 真生效 keys 数 (累积).
    /// applied < attempts × global_keys → 部分 partial.
    pub resubscribe_applied_keys: AtomicU64,

    // ===== v1.4.106 codex 1140 F8: 行情 push 投递失败计数 =====
    //
    // 之前 `bridge::push_parser` 用 `let _ = push_tx.try_send(event)` 静默吞错,
    // cache 已更新但 subscriber 收不到 push (audit Finding 8). 加 metric +
    // warn log 让 channel full / closed 立即可观测.
    /// CMD 6212 行情 push (BasicQot/OrderBook/Ticker/RT/KL/Broker/...) 因
    /// `push_tx` 队列满或关闭被 drop 的总次数 (累积).
    pub qot_push_dropped_total: AtomicU64,
    /// 按 SubType 拆分的 drop 计数 (proto Qot_Common.SubType: 0..=17, 共 18 桶).
    /// 桶 0 = "未知 / 不属于任何已知 SubType" (兜底, e.g. 拼错的 sub_type).
    /// 桶 1..=17 = 对应 SubType. v1.4.106 codex 1140 F8 加.
    pub qot_push_dropped_by_sub_type: [AtomicU64; 18],

    // ===== KeepAlive 指标 =====
    /// KeepAlive 超时断开数
    pub keepalive_timeouts: AtomicU64,

    // ===== 延迟采样 =====
    /// 最近 N 个请求延迟的环形缓冲 (纳秒)
    latency_ring: RwLock<LatencyRing>,
}

/// 延迟环形缓冲 — 保留最近 1000 个采样
struct LatencyRing {
    buf: Vec<u64>,
    pos: usize,
    count: u64,
    total_ns: u64,
}

const LATENCY_RING_SIZE: usize = 1000;

impl LatencyRing {
    fn new() -> Self {
        Self {
            buf: vec![0u64; LATENCY_RING_SIZE],
            pos: 0,
            count: 0,
            total_ns: 0,
        }
    }

    fn push(&mut self, ns: u64) {
        // 减去被覆盖的旧值
        if self.count >= LATENCY_RING_SIZE as u64 {
            self.total_ns = self.total_ns.saturating_sub(self.buf[self.pos]);
        }
        self.buf[self.pos] = ns;
        self.total_ns += ns;
        self.pos = (self.pos + 1) % LATENCY_RING_SIZE;
        self.count += 1;
    }

    fn stats(&self) -> LatencyStats {
        let n = self.count.min(LATENCY_RING_SIZE as u64) as usize;
        if n == 0 {
            return LatencyStats::default();
        }

        let mut samples: Vec<u64> = if self.count >= LATENCY_RING_SIZE as u64 {
            self.buf.clone()
        } else {
            self.buf[..n].to_vec()
        };
        samples.sort_unstable();

        LatencyStats {
            count: self.count,
            avg_us: (self.total_ns / n as u64) / 1000,
            p50_us: samples[n / 2] / 1000,
            p95_us: samples[(n as f64 * 0.95) as usize] / 1000,
            p99_us: samples[(n as f64 * 0.99).min((n - 1) as f64) as usize] / 1000,
            max_us: samples[n - 1] / 1000,
        }
    }
}

/// 延迟统计摘要 (微秒)
#[derive(Default)]
pub struct LatencyStats {
    /// 总采样数
    pub count: u64,
    /// 平均延迟 (微秒)
    pub avg_us: u64,
    /// P50 延迟
    pub p50_us: u64,
    /// P95 延迟
    pub p95_us: u64,
    /// P99 延迟
    pub p99_us: u64,
    /// 最大延迟
    pub max_us: u64,
}

/// v1.4.84 §14: 把 24 小时 counters snapshot 格式化为空格分隔的单行.
///
/// 输出格式: `"h00=N h01=N ... h23=N"` — 便于人眼 scan 时段异常,
/// 同时保持 parseable (awk / grep / Prometheus textfile).
fn format_hour_row(hb: &HourBreakdown) -> String {
    let snap = hb.snapshot();
    let mut out = String::with_capacity(24 * 10);
    for (i, v) in snap.iter().enumerate() {
        if i > 0 {
            out.push(' ');
        }
        out.push_str(&format!("h{:02}={}", i, v));
    }
    out
}

fn qot_sub_type_bucket(sub_type: i32) -> usize {
    if (0..18).contains(&sub_type) {
        sub_type as usize
    } else {
        0
    }
}

impl GatewayMetrics {
    pub fn new() -> Self {
        Self {
            start_time: Instant::now(),
            total_connections: AtomicU64::new(0),
            total_disconnections: AtomicU64::new(0),
            rejected_connections: AtomicU64::new(0),
            total_requests: AtomicU64::new(0),
            total_request_errors: AtomicU64::new(0),
            total_response_bytes: AtomicU64::new(0),
            backend_reconnects: AtomicU64::new(0),
            backend_reconnect_failures: AtomicU64::new(0),
            last_reconnect_ms: AtomicU64::new(0),
            backend_online: AtomicU64::new(1),
            backend_pushes_received: AtomicU64::new(0),
            client_pushes_sent: AtomicU64::new(0),
            client_push_send_failures: AtomicU64::new(0),
            qot_client_push_backpressure_drops: AtomicU64::new(0),
            qot_client_push_backpressure_drops_by_sub_type: [const { AtomicU64::new(0) }; 18],
            backend_pushes_cmd_quote: AtomicU64::new(0),
            backend_pushes_cmd_trade_legacy: AtomicU64::new(0),
            backend_pushes_cmd_trade_new: AtomicU64::new(0),
            backend_pushes_cmd_msg_center: AtomicU64::new(0),
            backend_pushes_cmd_other: AtomicU64::new(0),
            backend_pushes_cmd_quote_by_hour: HourBreakdown::new(),
            backend_pushes_cmd_trade_legacy_by_hour: HourBreakdown::new(),
            backend_pushes_cmd_trade_new_by_hour: HourBreakdown::new(),
            backend_pushes_cmd_msg_center_by_hour: HourBreakdown::new(),
            qot_subscribe_ops: AtomicU64::new(0),
            qot_unsubscribe_ops: AtomicU64::new(0),
            cold_cache_wait_total: AtomicU64::new(0),
            cold_cache_wait_hit: AtomicU64::new(0),
            cold_cache_wait_timeout: AtomicU64::new(0),
            resubscribe_ops: AtomicU64::new(0),
            resubscribe_attempts: AtomicU64::new(0),
            resubscribe_applied_keys: AtomicU64::new(0),
            // v1.4.106 codex 1140 F8: qot push drop counter init.
            qot_push_dropped_total: AtomicU64::new(0),
            qot_push_dropped_by_sub_type: [const { AtomicU64::new(0) }; 18],
            keepalive_timeouts: AtomicU64::new(0),
            latency_ring: RwLock::new(LatencyRing::new()),
        }
    }

    /// 记录一次请求延迟 (纳秒)
    pub fn record_latency_ns(&self, ns: u64) {
        self.latency_ring.write().push(ns);
    }

    /// v1.4.106 codex 1140 F8: 记录一次 qot push 被 drop (channel full / closed).
    ///
    /// `sub_type` 范围 0..=17 (proto Qot_Common.SubType). 越界值归桶 0
    /// (未知). 同时 bump 总计数 + per-sub-type 桶, 保证 dashboard 可分维度.
    pub fn record_qot_push_dropped(&self, sub_type: i32) {
        self.qot_push_dropped_total.fetch_add(1, Ordering::Relaxed);
        let bucket = qot_sub_type_bucket(sub_type);
        self.qot_push_dropped_by_sub_type[bucket].fetch_add(1, Ordering::Relaxed);
    }

    /// 记录一次 QOT fanout 因单个客户端 channel 满而丢给该客户端的帧。
    pub fn record_qot_client_push_backpressure_drop(&self, sub_type: i32) {
        self.qot_client_push_backpressure_drops
            .fetch_add(1, Ordering::Relaxed);
        let bucket = qot_sub_type_bucket(sub_type);
        self.qot_client_push_backpressure_drops_by_sub_type[bucket].fetch_add(1, Ordering::Relaxed);
    }

    /// v1.4.106 codex 1140 F8: 读取每个 sub_type 桶的 drop 计数 (snapshot).
    /// 用于 metrics endpoint render.
    pub fn qot_push_dropped_per_sub_type(&self) -> [u64; 18] {
        let mut out = [0u64; 18];
        for (i, slot) in self.qot_push_dropped_by_sub_type.iter().enumerate() {
            out[i] = slot.load(Ordering::Relaxed);
        }
        out
    }

    pub fn qot_client_push_backpressure_drops_per_sub_type(&self) -> [u64; 18] {
        let mut out = [0u64; 18];
        for (i, slot) in self
            .qot_client_push_backpressure_drops_by_sub_type
            .iter()
            .enumerate()
        {
            out[i] = slot.load(Ordering::Relaxed);
        }
        out
    }

    /// 获取延迟统计
    pub fn latency_stats(&self) -> LatencyStats {
        self.latency_ring.read().stats()
    }

    /// 格式化运行时间
    pub fn uptime_str(&self) -> String {
        let elapsed = self.start_time.elapsed();
        let secs = elapsed.as_secs();
        let days = secs / 86400;
        let hours = (secs % 86400) / 3600;
        let mins = (secs % 3600) / 60;
        let s = secs % 60;
        if days > 0 {
            format!("{days}d {hours}h {mins}m {s}s")
        } else if hours > 0 {
            format!("{hours}h {mins}m {s}s")
        } else {
            format!("{mins}m {s}s")
        }
    }

    /// 生成 telnet 可展示的指标报告
    pub fn report(&self) -> String {
        let lat = self.latency_stats();
        let backend_status = if self.backend_online.load(Ordering::Relaxed) == 1 {
            "ONLINE"
        } else {
            "OFFLINE"
        };

        let total_req = self.total_requests.load(Ordering::Relaxed);
        let uptime_secs = self.start_time.elapsed().as_secs_f64();
        let avg_rps = if uptime_secs > 0.0 {
            total_req as f64 / uptime_secs
        } else {
            0.0
        };

        format!(
            "=== Gateway Metrics ===\r\n\
             Uptime: {uptime}\r\n\
             \r\n\
             [Connections]\r\n\
             total_accepted: {total_conn}\r\n\
             total_disconnected: {total_disconn}\r\n\
             rejected (limit): {rejected}\r\n\
             keepalive_timeouts: {ka_timeout}\r\n\
             \r\n\
             [Requests]\r\n\
             total_requests: {total_req}\r\n\
             total_errors: {total_err}\r\n\
             avg_rps: {avg_rps:.1}\r\n\
             response_bytes: {resp_bytes}\r\n\
             \r\n\
             [Latency (recent {lat_count} samples)]\r\n\
             avg: {lat_avg}us  p50: {lat_p50}us  p95: {lat_p95}us  p99: {lat_p99}us  max: {lat_max}us\r\n\
             \r\n\
             [Backend]\r\n\
             status: {backend_status}\r\n\
             reconnects: {reconnects}\r\n\
             reconnect_failures: {reconnect_fail}\r\n\
             pushes_received: {push_recv}\r\n\
             pushes_sent_to_clients: {push_sent}\r\n\
             push_send_failures_to_clients: {push_send_failures}\r\n\
             qot_client_push_backpressure_drops: {qot_client_backpressure_drops}\r\n\
             \r\n\
             [Pushes by CMD (v1.4.83 §14)]\r\n\
             cmd_6212_quote: {push_cmd_quote}\r\n\
             cmd_4716_trade_legacy: {push_cmd_trade_legacy}\r\n\
             cmd_14716_trade_new: {push_cmd_trade_new}\r\n\
             cmd_5300_msg_center: {push_cmd_msg_center}\r\n\
             cmd_other: {push_cmd_other}\r\n\
             \r\n\
             [Pushes by CMD × UTC hour (v1.4.84 §14)]\r\n\
             cmd_14716_trade_new_hour_0..23: {hour_trade_new}\r\n\
             cmd_6212_quote_hour_0..23: {hour_quote}\r\n\
             cmd_4716_trade_legacy_hour_0..23: {hour_trade_legacy}\r\n\
             cmd_5300_msg_center_hour_0..23: {hour_msg_center}\r\n\
             \r\n\
             [Subscriptions]\r\n\
             subscribe_ops: {sub_ops}\r\n\
             unsubscribe_ops: {unsub_ops}\r\n\
             resubscribe_ops: {resub_ops}\r\n\
             \r\n\
             [Cold-cache wait (v1.4.110 §P3 #19)]\r\n\
             total: {cc_total}  hit: {cc_hit}  timeout: {cc_timeout}\r\n",
            uptime = self.uptime_str(),
            total_conn = self.total_connections.load(Ordering::Relaxed),
            total_disconn = self.total_disconnections.load(Ordering::Relaxed),
            rejected = self.rejected_connections.load(Ordering::Relaxed),
            ka_timeout = self.keepalive_timeouts.load(Ordering::Relaxed),
            total_req = total_req,
            total_err = self.total_request_errors.load(Ordering::Relaxed),
            resp_bytes = self.total_response_bytes.load(Ordering::Relaxed),
            lat_count = lat.count.min(LATENCY_RING_SIZE as u64),
            lat_avg = lat.avg_us,
            lat_p50 = lat.p50_us,
            lat_p95 = lat.p95_us,
            lat_p99 = lat.p99_us,
            lat_max = lat.max_us,
            reconnects = self.backend_reconnects.load(Ordering::Relaxed),
            reconnect_fail = self.backend_reconnect_failures.load(Ordering::Relaxed),
            push_recv = self.backend_pushes_received.load(Ordering::Relaxed),
            push_sent = self.client_pushes_sent.load(Ordering::Relaxed),
            push_send_failures = self.client_push_send_failures.load(Ordering::Relaxed),
            qot_client_backpressure_drops = self
                .qot_client_push_backpressure_drops
                .load(Ordering::Relaxed),
            push_cmd_quote = self.backend_pushes_cmd_quote.load(Ordering::Relaxed),
            push_cmd_trade_legacy = self.backend_pushes_cmd_trade_legacy.load(Ordering::Relaxed),
            push_cmd_trade_new = self.backend_pushes_cmd_trade_new.load(Ordering::Relaxed),
            push_cmd_msg_center = self.backend_pushes_cmd_msg_center.load(Ordering::Relaxed),
            push_cmd_other = self.backend_pushes_cmd_other.load(Ordering::Relaxed),
            hour_trade_new = format_hour_row(&self.backend_pushes_cmd_trade_new_by_hour),
            hour_quote = format_hour_row(&self.backend_pushes_cmd_quote_by_hour),
            hour_trade_legacy = format_hour_row(&self.backend_pushes_cmd_trade_legacy_by_hour),
            hour_msg_center = format_hour_row(&self.backend_pushes_cmd_msg_center_by_hour),
            sub_ops = self.qot_subscribe_ops.load(Ordering::Relaxed),
            unsub_ops = self.qot_unsubscribe_ops.load(Ordering::Relaxed),
            resub_ops = self.resubscribe_ops.load(Ordering::Relaxed),
            cc_total = self.cold_cache_wait_total.load(Ordering::Relaxed),
            cc_hit = self.cold_cache_wait_hit.load(Ordering::Relaxed),
            cc_timeout = self.cold_cache_wait_timeout.load(Ordering::Relaxed),
        )
    }
}

impl Default for GatewayMetrics {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod tests;