1use std::sync::atomic::{AtomicU64, Ordering};
10use std::time::Instant;
11
12use chrono::{Timelike, Utc};
13use parking_lot::RwLock;
14
15mod prometheus;
16pub use prometheus::install_prometheus_extension;
17
18#[derive(Debug)]
23pub struct HourBreakdown {
24 counters: [AtomicU64; 24],
25}
26
27impl HourBreakdown {
28 pub const fn new() -> Self {
29 Self {
30 counters: [const { AtomicU64::new(0) }; 24],
31 }
32 }
33
34 pub fn bump_now(&self) {
36 let hour = Utc::now().hour() as usize;
37 if hour < 24 {
38 self.counters[hour].fetch_add(1, Ordering::Relaxed);
39 }
40 }
41
42 pub fn get(&self, hour: usize) -> u64 {
44 self.counters
45 .get(hour)
46 .map(|a| a.load(Ordering::Relaxed))
47 .unwrap_or(0)
48 }
49
50 pub fn snapshot(&self) -> [u64; 24] {
52 let mut out = [0u64; 24];
53 for (i, c) in self.counters.iter().enumerate() {
54 out[i] = c.load(Ordering::Relaxed);
55 }
56 out
57 }
58}
59
60impl Default for HourBreakdown {
61 fn default() -> Self {
62 Self::new()
63 }
64}
65
66pub struct GatewayMetrics {
68 pub start_time: Instant,
70
71 pub total_connections: AtomicU64,
74 pub total_disconnections: AtomicU64,
76 pub rejected_connections: AtomicU64,
78
79 pub total_requests: AtomicU64,
82 pub total_request_errors: AtomicU64,
84 pub total_response_bytes: AtomicU64,
86
87 pub backend_reconnects: AtomicU64,
90 pub backend_reconnect_failures: AtomicU64,
92 pub last_reconnect_ms: AtomicU64,
94 pub backend_online: AtomicU64,
96
97 pub backend_pushes_received: AtomicU64,
100 pub client_pushes_sent: AtomicU64,
102 pub client_push_send_failures: AtomicU64,
104 pub ordinary_client_push_backpressure_disconnects: AtomicU64,
108 pub qot_client_push_backpressure_drops: AtomicU64,
113 pub qot_client_push_backpressure_drops_by_sub_type: [AtomicU64; 18],
116 pub backend_pushes_cmd_quote: AtomicU64,
119 pub backend_pushes_cmd_trade_legacy: AtomicU64,
121 pub backend_pushes_cmd_trade_new: AtomicU64,
123 pub backend_pushes_cmd_msg_center: AtomicU64,
125 pub backend_pushes_cmd_other: AtomicU64,
127 pub backend_control_event_drops: AtomicU64,
130 pub message_center_async_dropped_total: AtomicU64,
133
134 pub backend_pushes_cmd_quote_by_hour: HourBreakdown,
138 pub backend_pushes_cmd_trade_legacy_by_hour: HourBreakdown,
140 pub backend_pushes_cmd_trade_new_by_hour: HourBreakdown,
142 pub backend_pushes_cmd_msg_center_by_hour: HourBreakdown,
144
145 pub qot_subscribe_ops: AtomicU64,
148 pub qot_unsubscribe_ops: AtomicU64,
150
151 pub cold_cache_wait_total: AtomicU64,
158 pub cold_cache_wait_hit: AtomicU64,
160 pub cold_cache_wait_timeout: AtomicU64,
162 pub resubscribe_ops: AtomicU64,
165
166 pub resubscribe_attempts: AtomicU64,
180 pub resubscribe_applied_keys: AtomicU64,
183
184 pub qot_push_dropped_total: AtomicU64,
192 pub qot_push_dropped_by_sub_type: [AtomicU64; 18],
196
197 pub keepalive_timeouts: AtomicU64,
200
201 latency_ring: RwLock<LatencyRing>,
204}
205
206struct LatencyRing {
208 buf: Vec<u64>,
209 pos: usize,
210 count: u64,
211 total_ns: u64,
212}
213
214const LATENCY_RING_SIZE: usize = 1000;
215
216impl LatencyRing {
217 fn new() -> Self {
218 Self {
219 buf: vec![0u64; LATENCY_RING_SIZE],
220 pos: 0,
221 count: 0,
222 total_ns: 0,
223 }
224 }
225
226 fn push(&mut self, ns: u64) {
227 if self.count >= LATENCY_RING_SIZE as u64 {
229 self.total_ns = self.total_ns.saturating_sub(self.buf[self.pos]);
230 }
231 self.buf[self.pos] = ns;
232 self.total_ns += ns;
233 self.pos = (self.pos + 1) % LATENCY_RING_SIZE;
234 self.count += 1;
235 }
236
237 fn stats(&self) -> LatencyStats {
238 let n = self.count.min(LATENCY_RING_SIZE as u64) as usize;
239 if n == 0 {
240 return LatencyStats::default();
241 }
242
243 let mut samples: Vec<u64> = if self.count >= LATENCY_RING_SIZE as u64 {
244 self.buf.clone()
245 } else {
246 self.buf[..n].to_vec()
247 };
248 samples.sort_unstable();
249
250 LatencyStats {
251 count: self.count,
252 avg_us: (self.total_ns / n as u64) / 1000,
253 p50_us: samples[n / 2] / 1000,
254 p95_us: samples[(n as f64 * 0.95) as usize] / 1000,
255 p99_us: samples[(n as f64 * 0.99).min((n - 1) as f64) as usize] / 1000,
256 max_us: samples[n - 1] / 1000,
257 }
258 }
259}
260
261#[derive(Default)]
263pub struct LatencyStats {
264 pub count: u64,
266 pub avg_us: u64,
268 pub p50_us: u64,
270 pub p95_us: u64,
272 pub p99_us: u64,
274 pub max_us: u64,
276}
277
278fn format_hour_row(hb: &HourBreakdown) -> String {
283 let snap = hb.snapshot();
284 let mut out = String::with_capacity(24 * 10);
285 for (i, v) in snap.iter().enumerate() {
286 if i > 0 {
287 out.push(' ');
288 }
289 out.push_str(&format!("h{:02}={}", i, v));
290 }
291 out
292}
293
294fn qot_sub_type_bucket(sub_type: i32) -> usize {
295 if (0..18).contains(&sub_type) {
296 sub_type as usize
297 } else {
298 0
299 }
300}
301
302impl GatewayMetrics {
303 pub fn new() -> Self {
304 Self {
305 start_time: Instant::now(),
306 total_connections: AtomicU64::new(0),
307 total_disconnections: AtomicU64::new(0),
308 rejected_connections: AtomicU64::new(0),
309 total_requests: AtomicU64::new(0),
310 total_request_errors: AtomicU64::new(0),
311 total_response_bytes: AtomicU64::new(0),
312 backend_reconnects: AtomicU64::new(0),
313 backend_reconnect_failures: AtomicU64::new(0),
314 last_reconnect_ms: AtomicU64::new(0),
315 backend_online: AtomicU64::new(1),
316 backend_pushes_received: AtomicU64::new(0),
317 client_pushes_sent: AtomicU64::new(0),
318 client_push_send_failures: AtomicU64::new(0),
319 ordinary_client_push_backpressure_disconnects: AtomicU64::new(0),
320 qot_client_push_backpressure_drops: AtomicU64::new(0),
321 qot_client_push_backpressure_drops_by_sub_type: [const { AtomicU64::new(0) }; 18],
322 backend_pushes_cmd_quote: AtomicU64::new(0),
323 backend_pushes_cmd_trade_legacy: AtomicU64::new(0),
324 backend_pushes_cmd_trade_new: AtomicU64::new(0),
325 backend_pushes_cmd_msg_center: AtomicU64::new(0),
326 backend_pushes_cmd_other: AtomicU64::new(0),
327 backend_control_event_drops: AtomicU64::new(0),
328 message_center_async_dropped_total: AtomicU64::new(0),
329 backend_pushes_cmd_quote_by_hour: HourBreakdown::new(),
330 backend_pushes_cmd_trade_legacy_by_hour: HourBreakdown::new(),
331 backend_pushes_cmd_trade_new_by_hour: HourBreakdown::new(),
332 backend_pushes_cmd_msg_center_by_hour: HourBreakdown::new(),
333 qot_subscribe_ops: AtomicU64::new(0),
334 qot_unsubscribe_ops: AtomicU64::new(0),
335 cold_cache_wait_total: AtomicU64::new(0),
336 cold_cache_wait_hit: AtomicU64::new(0),
337 cold_cache_wait_timeout: AtomicU64::new(0),
338 resubscribe_ops: AtomicU64::new(0),
339 resubscribe_attempts: AtomicU64::new(0),
340 resubscribe_applied_keys: AtomicU64::new(0),
341 qot_push_dropped_total: AtomicU64::new(0),
343 qot_push_dropped_by_sub_type: [const { AtomicU64::new(0) }; 18],
344 keepalive_timeouts: AtomicU64::new(0),
345 latency_ring: RwLock::new(LatencyRing::new()),
346 }
347 }
348
349 pub fn record_latency_ns(&self, ns: u64) {
351 self.latency_ring.write().push(ns);
352 }
353
354 pub fn record_qot_push_dropped(&self, sub_type: i32) {
359 self.qot_push_dropped_total.fetch_add(1, Ordering::Relaxed);
360 let bucket = qot_sub_type_bucket(sub_type);
361 self.qot_push_dropped_by_sub_type[bucket].fetch_add(1, Ordering::Relaxed);
362 }
363
364 pub fn record_qot_client_push_backpressure_drop(&self, sub_type: i32) {
366 self.qot_client_push_backpressure_drops
367 .fetch_add(1, Ordering::Relaxed);
368 let bucket = qot_sub_type_bucket(sub_type);
369 self.qot_client_push_backpressure_drops_by_sub_type[bucket].fetch_add(1, Ordering::Relaxed);
370 }
371
372 pub fn qot_push_dropped_per_sub_type(&self) -> [u64; 18] {
375 let mut out = [0u64; 18];
376 for (i, slot) in self.qot_push_dropped_by_sub_type.iter().enumerate() {
377 out[i] = slot.load(Ordering::Relaxed);
378 }
379 out
380 }
381
382 pub fn qot_client_push_backpressure_drops_per_sub_type(&self) -> [u64; 18] {
383 let mut out = [0u64; 18];
384 for (i, slot) in self
385 .qot_client_push_backpressure_drops_by_sub_type
386 .iter()
387 .enumerate()
388 {
389 out[i] = slot.load(Ordering::Relaxed);
390 }
391 out
392 }
393
394 pub fn latency_stats(&self) -> LatencyStats {
396 self.latency_ring.read().stats()
397 }
398
399 pub fn uptime_str(&self) -> String {
401 let elapsed = self.start_time.elapsed();
402 let secs = elapsed.as_secs();
403 let days = secs / 86400;
404 let hours = (secs % 86400) / 3600;
405 let mins = (secs % 3600) / 60;
406 let s = secs % 60;
407 if days > 0 {
408 format!("{days}d {hours}h {mins}m {s}s")
409 } else if hours > 0 {
410 format!("{hours}h {mins}m {s}s")
411 } else {
412 format!("{mins}m {s}s")
413 }
414 }
415
416 pub fn report(&self) -> String {
418 let lat = self.latency_stats();
419 let backend_status = if self.backend_online.load(Ordering::Relaxed) == 1 {
420 "ONLINE"
421 } else {
422 "OFFLINE"
423 };
424
425 let total_req = self.total_requests.load(Ordering::Relaxed);
426 let uptime_secs = self.start_time.elapsed().as_secs_f64();
427 let avg_rps = if uptime_secs > 0.0 {
428 total_req as f64 / uptime_secs
429 } else {
430 0.0
431 };
432
433 format!(
434 "=== Gateway Metrics ===\r\n\
435 Uptime: {uptime}\r\n\
436 \r\n\
437 [Connections]\r\n\
438 total_accepted: {total_conn}\r\n\
439 total_disconnected: {total_disconn}\r\n\
440 rejected (limit): {rejected}\r\n\
441 keepalive_timeouts: {ka_timeout}\r\n\
442 \r\n\
443 [Requests]\r\n\
444 total_requests: {total_req}\r\n\
445 total_errors: {total_err}\r\n\
446 avg_rps: {avg_rps:.1}\r\n\
447 response_bytes: {resp_bytes}\r\n\
448 \r\n\
449 [Latency (recent {lat_count} samples)]\r\n\
450 avg: {lat_avg}us p50: {lat_p50}us p95: {lat_p95}us p99: {lat_p99}us max: {lat_max}us\r\n\
451 \r\n\
452 [Backend]\r\n\
453 status: {backend_status}\r\n\
454 reconnects: {reconnects}\r\n\
455 reconnect_failures: {reconnect_fail}\r\n\
456 pushes_received: {push_recv}\r\n\
457 pushes_sent_to_clients: {push_sent}\r\n\
458 push_send_failures_to_clients: {push_send_failures}\r\n\
459 ordinary_client_push_backpressure_disconnects: {ordinary_client_backpressure_disconnects}\r\n\
460 qot_client_push_backpressure_drops: {qot_client_backpressure_drops}\r\n\
461 backend_control_event_drops: {backend_control_event_drops}\r\n\
462 message_center_async_drops: {message_center_async_drops}\r\n\
463 \r\n\
464 [Pushes by CMD (v1.4.83 §14)]\r\n\
465 cmd_6212_quote: {push_cmd_quote}\r\n\
466 cmd_4716_trade_legacy: {push_cmd_trade_legacy}\r\n\
467 cmd_14716_trade_new: {push_cmd_trade_new}\r\n\
468 cmd_5300_msg_center: {push_cmd_msg_center}\r\n\
469 cmd_other: {push_cmd_other}\r\n\
470 \r\n\
471 [Pushes by CMD × UTC hour (v1.4.84 §14)]\r\n\
472 cmd_14716_trade_new_hour_0..23: {hour_trade_new}\r\n\
473 cmd_6212_quote_hour_0..23: {hour_quote}\r\n\
474 cmd_4716_trade_legacy_hour_0..23: {hour_trade_legacy}\r\n\
475 cmd_5300_msg_center_hour_0..23: {hour_msg_center}\r\n\
476 \r\n\
477 [Subscriptions]\r\n\
478 subscribe_ops: {sub_ops}\r\n\
479 unsubscribe_ops: {unsub_ops}\r\n\
480 resubscribe_ops: {resub_ops}\r\n\
481 \r\n\
482 [Cold-cache wait (v1.4.110 §P3 #19)]\r\n\
483 total: {cc_total} hit: {cc_hit} timeout: {cc_timeout}\r\n",
484 uptime = self.uptime_str(),
485 total_conn = self.total_connections.load(Ordering::Relaxed),
486 total_disconn = self.total_disconnections.load(Ordering::Relaxed),
487 rejected = self.rejected_connections.load(Ordering::Relaxed),
488 ka_timeout = self.keepalive_timeouts.load(Ordering::Relaxed),
489 total_req = total_req,
490 total_err = self.total_request_errors.load(Ordering::Relaxed),
491 resp_bytes = self.total_response_bytes.load(Ordering::Relaxed),
492 lat_count = lat.count.min(LATENCY_RING_SIZE as u64),
493 lat_avg = lat.avg_us,
494 lat_p50 = lat.p50_us,
495 lat_p95 = lat.p95_us,
496 lat_p99 = lat.p99_us,
497 lat_max = lat.max_us,
498 reconnects = self.backend_reconnects.load(Ordering::Relaxed),
499 reconnect_fail = self.backend_reconnect_failures.load(Ordering::Relaxed),
500 push_recv = self.backend_pushes_received.load(Ordering::Relaxed),
501 push_sent = self.client_pushes_sent.load(Ordering::Relaxed),
502 push_send_failures = self.client_push_send_failures.load(Ordering::Relaxed),
503 ordinary_client_backpressure_disconnects = self
504 .ordinary_client_push_backpressure_disconnects
505 .load(Ordering::Relaxed),
506 qot_client_backpressure_drops = self
507 .qot_client_push_backpressure_drops
508 .load(Ordering::Relaxed),
509 backend_control_event_drops = self.backend_control_event_drops.load(Ordering::Relaxed),
510 message_center_async_drops = self
511 .message_center_async_dropped_total
512 .load(Ordering::Relaxed),
513 push_cmd_quote = self.backend_pushes_cmd_quote.load(Ordering::Relaxed),
514 push_cmd_trade_legacy = self.backend_pushes_cmd_trade_legacy.load(Ordering::Relaxed),
515 push_cmd_trade_new = self.backend_pushes_cmd_trade_new.load(Ordering::Relaxed),
516 push_cmd_msg_center = self.backend_pushes_cmd_msg_center.load(Ordering::Relaxed),
517 push_cmd_other = self.backend_pushes_cmd_other.load(Ordering::Relaxed),
518 hour_trade_new = format_hour_row(&self.backend_pushes_cmd_trade_new_by_hour),
519 hour_quote = format_hour_row(&self.backend_pushes_cmd_quote_by_hour),
520 hour_trade_legacy = format_hour_row(&self.backend_pushes_cmd_trade_legacy_by_hour),
521 hour_msg_center = format_hour_row(&self.backend_pushes_cmd_msg_center_by_hour),
522 sub_ops = self.qot_subscribe_ops.load(Ordering::Relaxed),
523 unsub_ops = self.qot_unsubscribe_ops.load(Ordering::Relaxed),
524 resub_ops = self.resubscribe_ops.load(Ordering::Relaxed),
525 cc_total = self.cold_cache_wait_total.load(Ordering::Relaxed),
526 cc_hit = self.cold_cache_wait_hit.load(Ordering::Relaxed),
527 cc_timeout = self.cold_cache_wait_timeout.load(Ordering::Relaxed),
528 )
529 }
530}
531
532impl Default for GatewayMetrics {
533 fn default() -> Self {
534 Self::new()
535 }
536}
537
538#[cfg(test)]
539mod tests;