1use std::{sync::Arc, time::Duration};
13
14use futu_server::listener::ApiServer;
15use futu_server::push::PushDispatcher;
16
17use super::{
18 GatewayBridge, PushEvent, build_order_fill_update_push, build_order_update_push,
19 keep_subscribe_quotes, resolve_push_header, resubscribe_quotes, sync_current_quote_desired_set,
20};
21
22#[cfg(test)]
23mod tests;
24mod trade_requery;
25mod trade_requery_worker;
26
27use trade_requery_worker::{
28 TRADE_REQUERY_WORKER_QUEUE_CAPACITY, collect_cached_order_backend_ids,
29 enqueue_trade_requery_event, handle_trade_requery_worker_event,
30 record_place_order_update_push_after_dispatch, spawn_account_asset_refresh_after_trade_notice,
31};
32
33const TRADE_NOTICE_ORDER_NTF: u32 = 100;
36const TRADE_NOTICE_ORDER_FILL_UPDATE: u32 = 6;
39impl GatewayBridge {
40 pub fn create_push_dispatcher(
42 &self,
43 server: &ApiServer,
44 external_sinks: Vec<Arc<dyn futu_server::push::ExternalPushSink>>,
45 ) -> PushDispatcher {
46 let mut dispatcher = PushDispatcher::new(
47 Arc::clone(server.connections()),
48 self.subscription_runtime().manager(),
49 )
50 .with_metrics(Arc::clone(self.push_runtime().metrics()));
51 for sink in external_sinks {
52 dispatcher = dispatcher.with_external_sink(sink);
53 }
54 dispatcher
55 }
56
57 pub fn start_push_dispatcher(
61 &self,
62 server: &ApiServer,
63 mut push_rx: tokio::sync::mpsc::Receiver<PushEvent>,
64 external_sinks: Vec<Arc<dyn futu_server::push::ExternalPushSink>>,
65 ) {
66 let dispatcher = Arc::new(self.create_push_dispatcher(server, external_sinks));
67 let caches = self.caches();
68 let broker_runtime_for_dispatch = self.broker_runtime().clone();
69 let trd_cache = Arc::clone(&caches.trd_cache);
70 let push_health_for_dispatch = Arc::clone(self.push_runtime().push_health());
72 let background_task_tracker_for_dispatch = self.background_runtime().tracker();
73 let (trade_requery_tx, mut trade_requery_rx) =
74 tokio::sync::mpsc::channel::<PushEvent>(TRADE_REQUERY_WORKER_QUEUE_CAPACITY);
75 {
76 let dispatcher_for_trade_requery = Arc::clone(&dispatcher);
77 let broker_runtime_for_trade_requery = broker_runtime_for_dispatch.clone();
78 let trd_cache_for_trade_requery = Arc::clone(&trd_cache);
79 let push_health_for_trade_requery = Arc::clone(&push_health_for_dispatch);
80 self.background_runtime().track(tokio::spawn(async move {
81 tracing::info!(
82 capacity = TRADE_REQUERY_WORKER_QUEUE_CAPACITY,
83 "trade requery worker task started"
84 );
85 while let Some(event) = trade_requery_rx.recv().await {
86 handle_trade_requery_worker_event(
87 Arc::clone(&dispatcher_for_trade_requery),
88 broker_runtime_for_trade_requery.clone(),
89 Arc::clone(&trd_cache_for_trade_requery),
90 Arc::clone(&push_health_for_trade_requery),
91 event,
92 )
93 .await;
94 }
95 tracing::warn!("trade requery worker task ended");
96 }));
97 }
98 {
100 let push_health_for_staleness = Arc::clone(self.push_runtime().push_health());
101 let subscriptions_for_staleness = self.subscription_runtime().manager();
102 let static_cache_for_staleness = Arc::clone(&caches.static_cache);
103 let qot_right_cache_for_staleness = Arc::clone(&caches.qot_right_cache);
104 let broker_runtime_for_staleness = self.broker_runtime().clone();
105 let metrics_for_staleness = Arc::clone(self.push_runtime().metrics());
108 self.background_runtime().track(tokio::spawn(async move {
109 tracing::info!(
110 "v1.4.83 §9 F3/F4: push-stream staleness detector started \
111 (interval=30s, stale_threshold=60s, F4 trip if re-sub fails within 60s)"
112 );
113 let mut interval = tokio::time::interval(std::time::Duration::from_secs(30));
114 interval.tick().await;
116 let mut last_resub_at_ms: i64 = 0;
119 loop {
120 interval.tick().await;
121 let snap = push_health_for_staleness.snapshot();
122 if snap.last_push_received_at_ms == 0 {
124 continue;
125 }
126 let now_ms = crate::bridge::clock::wall_clock_since_unix_epoch_or_zero(
127 "push staleness detector",
128 )
129 .as_millis() as i64;
130 let stale_ms = now_ms.saturating_sub(snap.last_push_received_at_ms);
131 if stale_ms < 60_000 {
132 last_resub_at_ms = 0;
134 continue;
135 }
136 let qot_conn_ids = subscriptions_for_staleness.get_all_qot_conn_ids();
144 let trd_conn_ids = subscriptions_for_staleness.get_all_trd_conn_ids();
145 if qot_conn_ids.is_empty() && trd_conn_ids.is_empty() {
146 continue;
148 }
149 if last_resub_at_ms != 0 && now_ms.saturating_sub(last_resub_at_ms) < 60_000 {
151 push_health_for_staleness.trip_circuit();
152 tracing::error!(
153 stale_ms,
154 since_last_resub_ms = now_ms - last_resub_at_ms,
155 qot_conns = qot_conn_ids.len(),
156 trd_conns = trd_conn_ids.len(),
157 "v1.4.83 §9 F4: push stream still stale after recent re-subscribe, \
158 circuit tripped (30s cooldown, dispatcher skips)"
159 );
160 continue;
161 }
162 if qot_conn_ids.is_empty() {
167 tracing::warn!(
168 stale_ms,
169 trd_conns = trd_conn_ids.len(),
170 "v1.4.106 audit 0932 F5: trade-only push stream stale >60s — \
171 no trade re-subscribe protocol exists; recommend manual reconnect \
172 if persists (will accumulate to F4 circuit trip on next cycle)"
173 );
174 continue;
176 }
177 let Some(be) = broker_runtime_for_staleness.platform_backend() else {
179 tracing::warn!(
180 "v1.4.83 §9 F3: stale detected but backend unavailable, \
181 retry next cycle"
182 );
183 continue;
184 };
185 push_health_for_staleness.record_resubscribe();
186 last_resub_at_ms = now_ms;
187 let outcome = resubscribe_quotes(
188 &be,
189 &subscriptions_for_staleness,
190 &static_cache_for_staleness,
191 &qot_right_cache_for_staleness.get(),
192 )
193 .await;
194 if outcome.is_partial {
197 push_health_for_staleness
198 .record_resubscribe_partial(outcome.missed_keys.len() as u32);
199 }
200 metrics_for_staleness
204 .resubscribe_attempts
205 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
206 metrics_for_staleness.resubscribe_applied_keys.fetch_add(
207 outcome.applied_keys as u64,
208 std::sync::atomic::Ordering::Relaxed,
209 );
210 metrics_for_staleness.resubscribe_ops.fetch_add(
213 outcome.applied_keys as u64,
214 std::sync::atomic::Ordering::Relaxed,
215 );
216 tracing::warn!(
217 stale_ms,
218 attempted = outcome.attempted_keys,
219 applied = outcome.applied_keys,
220 missed = outcome.missed_keys.len(),
221 is_partial = outcome.is_partial,
222 active_qot_conns = qot_conn_ids.len(),
223 active_trd_conns = trd_conn_ids.len(),
224 "v1.4.83 §9 F3 + v1.4.106 audit 0631 F4: push stream stale \
225 >60s with active subscribers — auto re-subscribed quotes"
226 );
227 }
228 }));
229 }
230 {
235 let subscriptions_for_keep_sub = self.subscription_runtime().manager();
236 let static_cache_for_keep_sub = Arc::clone(&caches.static_cache);
237 let qot_right_cache_for_keep_sub = Arc::clone(&caches.qot_right_cache);
238 let broker_runtime_for_keep_sub = self.broker_runtime().clone();
239 self.background_runtime().track(tokio::spawn(async move {
240 tracing::info!(
241 "quote keep-sub status task started \
242 (interval=120s, C++ MktQotSub KeepSubStatusPause_S)"
243 );
244 let mut interval = tokio::time::interval(Duration::from_secs(120));
245 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
248 interval.tick().await;
251 loop {
252 interval.tick().await;
253 let active_keys = subscriptions_for_keep_sub.qot_global_desired_keys().len();
254 if active_keys == 0 {
255 continue;
256 }
257 let Some(be) = broker_runtime_for_keep_sub.platform_backend() else {
258 tracing::warn!(
259 active_keys,
260 "v1.4.112: quote keep-sub skipped because platform backend is unavailable"
261 );
262 continue;
263 };
264 let outcome = keep_subscribe_quotes(
265 &be,
266 &subscriptions_for_keep_sub,
267 &static_cache_for_keep_sub,
268 &qot_right_cache_for_keep_sub.get(),
269 )
270 .await;
271 if outcome.is_partial {
272 tracing::warn!(
273 active_keys,
274 attempted = outcome.attempted_keys,
275 applied = outcome.applied_keys,
276 missed = outcome.missed_keys.len(),
277 error = ?outcome.error_reason,
278 "v1.4.112: quote keep-sub completed with partial outcome"
279 );
280 } else {
281 tracing::debug!(
282 active_keys,
283 attempted = outcome.attempted_keys,
284 applied = outcome.applied_keys,
285 "v1.4.112: quote keep-sub refreshed current desired set"
286 );
287 }
288 }
289 }));
290 }
291 {
299 let subscriptions_for_disconnect_sync = self.subscription_runtime().manager();
300 let static_cache_for_disconnect_sync = Arc::clone(&caches.static_cache);
301 let qot_right_cache_for_disconnect_sync = Arc::clone(&caches.qot_right_cache);
302 let broker_runtime_for_disconnect_sync = self.broker_runtime().clone();
303 self.background_runtime().track(tokio::spawn(async move {
304 let mut interval = tokio::time::interval(std::time::Duration::from_secs(5));
305 interval.tick().await;
306 let mut last_synced_generation =
307 subscriptions_for_disconnect_sync.qot_disconnect_sync_generation();
308 loop {
309 interval.tick().await;
310 let due = subscriptions_for_disconnect_sync.cleanup_due_disconnected_qot();
311 if !due.is_empty() {
312 tracing::info!(
313 due = ?due,
314 "v1.4.107: QOT disconnected conn cleanup became due; syncing backend desired set"
315 );
316 }
317
318 let generation =
319 subscriptions_for_disconnect_sync.qot_disconnect_sync_generation();
320 if generation == last_synced_generation {
321 continue;
322 }
323
324 let Some(be) = broker_runtime_for_disconnect_sync.platform_backend() else {
325 tracing::warn!(
326 generation,
327 last_synced_generation,
328 "v1.4.107: QOT disconnect cleanup pending but backend unavailable; retry next tick"
329 );
330 continue;
331 };
332
333 let outcome = sync_current_quote_desired_set(
334 &be,
335 &subscriptions_for_disconnect_sync,
336 &static_cache_for_disconnect_sync,
337 &qot_right_cache_for_disconnect_sync.get(),
338 &due,
339 )
340 .await;
341 if outcome.is_partial || outcome.error_reason.is_some() {
342 tracing::warn!(
343 generation,
344 error = ?outcome.error_reason,
345 missed = ?outcome.missed_keys,
346 "v1.4.107: QOT disconnect cleanup desired-set sync did not fully apply; retry next tick"
347 );
348 continue;
349 }
350 last_synced_generation = generation;
351 }
352 }));
353 }
354 {
357 let trd_cache_for_scan = Arc::clone(&caches.trd_cache);
358 let push_health_for_scan = Arc::clone(self.push_runtime().push_health());
359 self.background_runtime().track(tokio::spawn(async move {
360 tracing::info!(
361 "v1.4.83 §9 F6: orphan-order scan task started (interval=30s, threshold=300s)"
362 );
363 let mut interval = tokio::time::interval(std::time::Duration::from_secs(30));
364 interval.tick().await;
366 loop {
367 interval.tick().await;
368 let now_secs = crate::bridge::clock::wall_clock_since_unix_epoch_or_zero(
369 "orphan-order scan",
370 )
371 .as_secs_f64();
372 let orphans = trd_cache_for_scan.scan_orphan_orders(now_secs, 300.0);
373 push_health_for_scan.record_orphan_scan(orphans.len() as u64);
374 if !orphans.is_empty() {
375 for o in &orphans {
377 tracing::warn!(
378 acc_id = o.acc_id,
379 order_id = o.order_id,
380 order_id_ex = %o.order_id_ex,
381 code = %o.code,
382 age_secs = o.age_secs,
383 "v1.4.83 §9 F6 (v1.4.103 BUG-WUZONG-001 expanded): \
384 orphan order detected (status in 0/1/2/4 stub stuck, \
385 create_timestamp > 5min 前 — push 通道可能断流 / order 卡住)"
386 );
387 }
388 }
389 }
390 }));
391 }
392 self.background_runtime().track(tokio::spawn(async move {
393 tracing::info!("push dispatcher task started");
394 while let Some(event) = push_rx.recv().await {
395 if push_health_for_dispatch.should_skip_dispatch() {
399 tracing::warn!(
400 "v1.4.83 §9 F4: circuit tripped — skipping push event during cooldown"
401 );
402 continue;
403 }
404 match event {
405 PushEvent::QuotePush {
406 sec_key,
407 sub_type,
408 rehab_type,
409 proto_id,
410 body,
411 } => {
412 push_health_for_dispatch.record_qot_push_sub_type(sub_type);
413 dispatcher
417 .push_qot(&sec_key, sub_type, rehab_type, proto_id, body)
418 .await;
419 }
420 PushEvent::QuotePushToConn {
421 conn_id,
422 proto_id,
423 body,
424 } => {
425 dispatcher.push_qot_to_conn(conn_id, proto_id, body).await;
426 }
427 event @ (PushEvent::TradeReQuery { .. }
428 | PushEvent::CryptoOrderReQuery { .. }
429 | PushEvent::CryptoAssetReQuery { .. }) => {
430 enqueue_trade_requery_event(
431 &trade_requery_tx,
432 &push_health_for_dispatch,
433 event,
434 );
435 }
436 PushEvent::TradeOrderDirect { acc_id, orders } => {
437 let Some(acc) = trd_cache.lookup_account(acc_id) else {
441 tracing::warn!(
442 acc_id,
443 count = orders.len(),
444 "skip direct order notification — account cache miss, cannot derive C++ trd_env"
445 );
446 continue;
447 };
448 let trd_env = acc.trd_env;
449 let mut cached_orders = Vec::with_capacity(orders.len());
450 for raw_order in &orders {
451 match futu_backend::trade_query::order_proto_to_cached_like_cpp(
452 trd_env, raw_order,
453 ) {
454 Some(order) => {
455 if trd_cache.upsert_order(acc_id, order.clone()) {
456 cached_orders.push(order);
457 } else {
458 tracing::debug!(
459 acc_id,
460 order_id_ex = %order.order_id_ex,
461 order_version = order.order_version,
462 "skip direct order notification row — cache has same/newer order state"
463 );
464 }
465 }
466 None => {
467 tracing::warn!(
468 acc_id,
469 order_id = ?raw_order.order_id,
470 "skip direct order notification row — C++ UnPackOrderItem required fields missing or market invalid"
471 );
472 }
473 }
474 }
475 let order_ids_for_clear = collect_cached_order_backend_ids(&cached_orders);
476 if !order_ids_for_clear.is_empty() {
477 let cleared = trd_cache
478 .clear_pending_confirm_for_orders(acc_id, &order_ids_for_clear);
479 if cleared > 0 {
480 tracing::info!(
481 acc_id,
482 cleared,
483 order_ids_count = order_ids_for_clear.len(),
484 "direct order notify → cleared pending broker confirm flags"
485 );
486 }
487 }
488 for order in &cached_orders {
489 let Some((trd_env, trd_market)) =
490 resolve_push_header(&trd_cache, acc_id, order.trd_market)
491 else {
492 tracing::warn!(
493 acc_id,
494 order_id_ex = %order.order_id_ex,
495 order_market = ?order.trd_market,
496 "skip direct Trd_UpdateOrder push — account cache miss"
497 );
498 continue;
499 };
500 let push_body =
501 build_order_update_push(acc_id, trd_env, trd_market, order);
502 dispatcher
503 .push_trd_acc(
504 acc_id,
505 futu_core::proto_id::TRD_UPDATE_ORDER,
506 push_body,
507 )
508 .await;
509 record_place_order_update_push_after_dispatch(
510 order, trd_env, trd_market,
511 );
512 }
513 tracing::debug!(
514 acc_id,
515 count = cached_orders.len(),
516 notice_type = TRADE_NOTICE_ORDER_NTF,
517 "direct order notification applied"
518 );
519 }
520 PushEvent::TradeOrderCached { acc_id, orders } => {
521 for order in &orders {
522 let Some((trd_env, trd_market)) =
523 resolve_push_header(&trd_cache, acc_id, order.trd_market)
524 else {
525 tracing::warn!(
526 acc_id,
527 order_id_ex = %order.order_id_ex,
528 order_market = ?order.trd_market,
529 "skip cached Trd_UpdateOrder push — account cache miss"
530 );
531 continue;
532 };
533 let push_body =
534 build_order_update_push(acc_id, trd_env, trd_market, order);
535 dispatcher
536 .push_trd_acc(
537 acc_id,
538 futu_core::proto_id::TRD_UPDATE_ORDER,
539 push_body,
540 )
541 .await;
542 record_place_order_update_push_after_dispatch(
543 order, trd_env, trd_market,
544 );
545 }
546 }
547 PushEvent::TradeFillDirect { acc_id, fills } => {
548 let cleared = trd_cache.clear_pending_confirm_for_acc(acc_id);
552 if cleared > 0 {
553 tracing::info!(
554 acc_id,
555 cleared,
556 "direct fill notify → cleared pending broker confirm flags"
557 );
558 }
559 for fill in &fills {
560 let Some((trd_env, trd_market)) =
561 resolve_push_header(&trd_cache, acc_id, fill.trd_market)
562 else {
563 tracing::warn!(
564 acc_id,
565 fill_id_ex = %fill.fill_id_ex,
566 fill_market = ?fill.trd_market,
567 "skip direct Trd_UpdateOrderFill push — account cache miss"
568 );
569 continue;
570 };
571 let push_body =
572 build_order_fill_update_push(acc_id, trd_env, trd_market, fill);
573 dispatcher
574 .push_trd_acc(
575 acc_id,
576 futu_core::proto_id::TRD_UPDATE_ORDER_FILL,
577 push_body,
578 )
579 .await;
580 }
581 spawn_account_asset_refresh_after_trade_notice(
582 &background_task_tracker_for_dispatch,
583 broker_runtime_for_dispatch.clone(),
584 Arc::clone(&trd_cache),
585 acc_id,
586 TRADE_NOTICE_ORDER_FILL_UPDATE,
587 true,
588 );
589 }
590 PushEvent::BroadcastPush { proto_id, body } => {
591 dispatcher.push_broadcast(proto_id, body).await;
593 }
594 }
595 }
596 tracing::warn!("push dispatcher task ended");
597 }));
598 }
599}