fediverse_outbound_service.rs
574 lignes · 21841 octets
//! Outbound ActivityPub delivery queue + worker (RFC 0011). //! //! Persistent journal of activities that must be delivered to remote //! inboxes (auth `Accept{AuthRequest}`, invitations, lifecycle signals). //! The transport layer is abstracted behind `OutboundDelivery` so //! that RFC 0010 can plug in HTTP-Sig signing without disturbing the //! queue/worker plumbing. //! use async_trait::async_trait; use chrono::Utc; use sea_orm::{ ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, ExprTrait, QueryFilter, QueryOrder, Set, }; use std::time::Duration; use uuid::Uuid; use crate::models::fediverse_outbound_queue; use rustwarden_core::error::AppError; /// Lifecycle of an outbound queue row. /// /// In the Phase-2 slice livered by RFC 0011 only `Pending → Delivered` /// and `Pending → Failed` transitions occur. RFC 0010 introduces the /// retry/back-off ladder that uses `Failed → Pending` and the terminal /// `Failed → Dead`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OutboundStatus { /// Eligible for delivery. Pending, /// Successfully delivered. Delivered, /// Last attempt failed; not (yet) eligible for retry. Failed, /// Reached `max_attempts`, no further attempts will be made. Dead, } impl OutboundStatus { /// RFC 0010 retry ladder. The returned offset is added to `now` /// to compute `next_retry_at` after a failed delivery. Sequence: /// 5 min → 30 min → 2 h → 6 h → 24 h → 48 h, capped at 48 h. /// `attempts` is the **post-increment** value (number of failed /// attempts so far). pub fn next_retry_offset(attempts: i32) -> std::time::Duration { const LADDER_SECS: &[u64] = &[5 * 60, 30 * 60, 2 * 3600, 6 * 3600, 24 * 3600, 48 * 3600]; let bounded = std::cmp::Ord::max(attempts, 1) as usize - 1; let idx = bounded.min(LADDER_SECS.len() - 1); std::time::Duration::from_secs(LADDER_SECS[idx]) } /// Wire form stored in the `status` column. pub fn as_str(self) -> &'static str { match self { Self::Pending => "pending", Self::Delivered => "delivered", Self::Failed => "failed", Self::Dead => "dead", } } /// Best-effort parse — unknown values fall back to `Pending` so a /// data corruption never silently drops a row. pub fn parse(raw: &str) -> Self { match raw.trim().to_ascii_lowercase().as_str() { "delivered" => Self::Delivered, "failed" => Self::Failed, "dead" => Self::Dead, _ => Self::Pending, } } } /// Caller-supplied metadata used to enqueue an activity for delivery. #[derive(Debug, Clone)] pub struct EnqueueRequest { /// Canonical AP `id` of the activity (e.g. `https://a.tld/activities/uuid`). pub activity_id: String, /// Pre-rendered JSON-LD body that will be POSTed to `target_inbox`. pub payload_json: String, /// Inbox URL. pub target_inbox: String, /// Domain owning the inbox (used by trust checks). pub target_domain: String, /// Local actor on whose behalf the activity is signed. Nullable /// while only `NoopDelivery` is wired; required by RFC 0010. pub actor_user_id: Option<Uuid>, } /// Transport layer abstraction. /// /// The Phase-2 slice ships only [`NoopDelivery`]. RFC 0010 will swap in /// `HttpSignedDelivery` that uses RFC 0005 keys to sign requests. #[async_trait] pub trait OutboundDelivery: Send + Sync { /// Attempt to deliver `item` to its `target_inbox`. Returning `Ok` /// promotes the row to `Delivered`; returning `Err` flips it to /// `Failed` and increments `attempts`. async fn deliver(&self, item: &fediverse_outbound_queue::Model) -> Result<(), AppError>; } /// No-op transport for the slice livered by RFC 0011: pretends every /// delivery succeeded so the queue/worker plumbing can be exercised /// end-to-end without an HTTP stack. Production deployments must swap /// for `HttpSignedDelivery` once RFC 0010 lands. pub struct NoopDelivery; #[async_trait] impl OutboundDelivery for NoopDelivery { async fn deliver(&self, item: &fediverse_outbound_queue::Model) -> Result<(), AppError> { tracing::debug!( target = %item.target_inbox, activity = %item.activity_id, "NoopDelivery: pretending delivery succeeded", ); Ok(()) } } /// Stateless service exposing enqueue + drain operations on the /// outbound queue. pub struct FediverseOutboundService; impl FediverseOutboundService { /// Persist a new `pending` row. pub async fn enqueue( db: &DatabaseConnection, request: EnqueueRequest, ) -> Result<fediverse_outbound_queue::Model, AppError> { let now = Utc::now(); let model = fediverse_outbound_queue::ActiveModel { id: Set(Uuid::new_v4()), activity_id: Set(request.activity_id), payload_json: Set(request.payload_json), target_inbox: Set(request.target_inbox), target_domain: Set(request.target_domain), status: Set(OutboundStatus::Pending.as_str().to_string()), attempts: Set(0), max_attempts: Set(8), last_error: Set(None), next_retry_at: Set(None), created_at: Set(now), delivered_at: Set(None), actor_user_id: Set(request.actor_user_id), }; Ok(model.insert(db).await?) } /// Pick one eligible row, hand it to `transport`, and persist the /// outcome. Returns the updated row, or `None` if the queue is /// empty. pub async fn drain_one<T: OutboundDelivery>( db: &DatabaseConnection, transport: &T, ) -> Result<Option<fediverse_outbound_queue::Model>, AppError> { let now = Utc::now(); let candidate = fediverse_outbound_queue::Entity::find() .filter(fediverse_outbound_queue::Column::Status.eq(OutboundStatus::Pending.as_str())) .filter( fediverse_outbound_queue::Column::NextRetryAt .is_null() .or(fediverse_outbound_queue::Column::NextRetryAt.lte(now)), ) .order_by_asc(fediverse_outbound_queue::Column::CreatedAt) .one(db) .await?; let Some(model) = candidate else { return Ok(None); }; let delivery = transport.deliver(&model).await; let next_attempts = model.attempts + 1; let max_attempts = model.max_attempts; let model_for_update = model.clone(); let mut active: fediverse_outbound_queue::ActiveModel = model_for_update.into(); active.attempts = Set(next_attempts); match &delivery { Ok(()) => { active.status = Set(OutboundStatus::Delivered.as_str().to_string()); active.delivered_at = Set(Some(Utc::now())); active.last_error = Set(None); active.next_retry_at = Set(None); } Err(err) => { active.last_error = Set(Some(err.to_string())); if next_attempts >= max_attempts { // RFC 0010: terminal — no further retry. active.status = Set(OutboundStatus::Dead.as_str().to_string()); active.next_retry_at = Set(None); } else { // RFC 0010: stay `pending` and gate on `next_retry_at` // so the partial-index scan ignores the row until the // backoff has elapsed. let offset = OutboundStatus::next_retry_offset(next_attempts); let next = Utc::now() + chrono::Duration::from_std(offset).map_err(|e| { AppError::Internal(format!( "retry offset out of range for chrono::Duration: {e}", )) })?; active.status = Set(OutboundStatus::Pending.as_str().to_string()); active.next_retry_at = Set(Some(next)); } } } let updated = active.update(db).await?; // RFC 0019: structured tracing events — drive Prometheus / Loki / // OTEL exporters without coupling rustwarden-core to a metrics // crate. Gated by `app_settings.fediverse_observability_enabled` // (default `false`) — opt-in to avoid log noise on instances // that pipe nothing to an observability backend. let observability = is_fediverse_observability_enabled(db) .await .unwrap_or(false); if observability { match (&delivery, updated.status.as_str()) { (Ok(()), _) => { tracing::info!( target: "rustwarden::fediverse::outbound", activity_id = %updated.activity_id, target_inbox = %updated.target_inbox, target_domain = %updated.target_domain, attempts = updated.attempts, event = "delivered", ); } (Err(err), status) if status == OutboundStatus::Dead.as_str() => { tracing::error!( target: "rustwarden::fediverse::outbound", activity_id = %updated.activity_id, target_domain = %updated.target_domain, last_error = %err, attempts = updated.attempts, max_attempts = updated.max_attempts, event = "dead", ); } (Err(err), _) => { tracing::warn!( target: "rustwarden::fediverse::outbound", activity_id = %updated.activity_id, target_domain = %updated.target_domain, last_error = %err, attempts = updated.attempts, next_retry_at = ?updated.next_retry_at, event = "deferred", ); } } } Ok(Some(updated)) } } /// RFC 0019 + S4 observability toggle — `app_settings.fediverse_observability_enabled`, /// default **false**. When `false`, fediverse routes/services skip the /// structured `tracing::info!` / `warn!` / `error!` emit-points used for /// metric aggregation. Cheap (1 SELECT by key) ; cached by the upstream /// `AppSettingsService` if the consumer wires its standard cache. pub async fn is_fediverse_observability_enabled(db: &DatabaseConnection) -> Result<bool, AppError> { rustwarden_core::services::app_settings_service::AppSettingsService::get_bool( db, "fediverse_observability_enabled", false, ) .await } /// Background loop that periodically drains the queue. /// /// Spawned by the federation router builder (RFC 0011 hooks /// the spawn into the federation router builder in a follow-up patch — RFC 0006 /// will be the first consumer and triggers the spawn explicitly). pub struct OutboundWorker<T: OutboundDelivery> { db: DatabaseConnection, transport: T, poll_interval: Duration, } impl<T: OutboundDelivery> OutboundWorker<T> { /// Build a worker that polls every `poll_interval`. pub fn new(db: DatabaseConnection, transport: T, poll_interval: Duration) -> Self { Self { db, transport, poll_interval, } } /// Run the polling loop forever. Errors during a single delivery /// are logged but do not stop the loop. Cancellation is achieved /// by cancelling the surrounding task. pub async fn run(self) { loop { match FediverseOutboundService::drain_one(&self.db, &self.transport).await { Ok(Some(_)) => continue, // drain greedily while items exist Ok(None) => tokio::time::sleep(self.poll_interval).await, Err(err) => { tracing::error!( target: "rustwarden::fediverse::outbound", error = %err, event = "drain_failed", ); tokio::time::sleep(self.poll_interval).await; } } } } } #[cfg(test)] mod tests { use super::*; use sea_orm::{DatabaseBackend, MockDatabase, MockExecResult}; fn pending_row(id: Uuid) -> fediverse_outbound_queue::Model { let now = Utc::now(); fediverse_outbound_queue::Model { id, activity_id: format!("https://a.tld/activities/{id}"), payload_json: "{}".to_string(), target_inbox: "https://b.tld/inbox".to_string(), target_domain: "b.tld".to_string(), status: "pending".to_string(), attempts: 0, max_attempts: 8, last_error: None, next_retry_at: None, created_at: now, delivered_at: None, actor_user_id: None, } } struct ErringDelivery; #[async_trait] impl OutboundDelivery for ErringDelivery { async fn deliver(&self, _item: &fediverse_outbound_queue::Model) -> Result<(), AppError> { Err(AppError::Internal("simulated transport failure".into())) } } #[test] fn outbound_status_round_trips_through_string() { for s in [ OutboundStatus::Pending, OutboundStatus::Delivered, OutboundStatus::Failed, OutboundStatus::Dead, ] { assert_eq!(OutboundStatus::parse(s.as_str()), s); } // Unknown wire value falls back safely. assert_eq!(OutboundStatus::parse("garbage"), OutboundStatus::Pending); } #[test] fn enqueue_request_clone_works() { let r = EnqueueRequest { activity_id: "act".into(), payload_json: "{}".into(), target_inbox: "ib".into(), target_domain: "d".into(), actor_user_id: None, }; let c = r.clone(); assert_eq!(c.activity_id, "act"); } #[tokio::test] async fn enqueue_persists_pending_row() { let id = Uuid::new_v4(); let row = pending_row(id); let db: DatabaseConnection = MockDatabase::new(DatabaseBackend::Postgres) .append_query_results([vec![row.clone()]]) .into_connection(); let inserted = FediverseOutboundService::enqueue( &db, EnqueueRequest { activity_id: row.activity_id.clone(), payload_json: row.payload_json.clone(), target_inbox: row.target_inbox.clone(), target_domain: row.target_domain.clone(), actor_user_id: None, }, ) .await .expect("enqueue ok"); assert_eq!(inserted.status, "pending"); assert_eq!(inserted.attempts, 0); } #[tokio::test] async fn enqueue_uses_provided_activity_id() { let id = Uuid::new_v4(); let mut row = pending_row(id); row.activity_id = "https://example.test/specific-id".into(); let db: DatabaseConnection = MockDatabase::new(DatabaseBackend::Postgres) .append_query_results([vec![row.clone()]]) .into_connection(); let inserted = FediverseOutboundService::enqueue( &db, EnqueueRequest { activity_id: row.activity_id.clone(), payload_json: "{}".into(), target_inbox: row.target_inbox.clone(), target_domain: row.target_domain.clone(), actor_user_id: None, }, ) .await .unwrap(); assert_eq!(inserted.activity_id, "https://example.test/specific-id"); } #[tokio::test] async fn drain_one_returns_none_when_empty() { let db: DatabaseConnection = MockDatabase::new(DatabaseBackend::Postgres) .append_query_results([Vec::<fediverse_outbound_queue::Model>::new()]) .into_connection(); let result = FediverseOutboundService::drain_one(&db, &NoopDelivery) .await .unwrap(); assert!(result.is_none()); } #[tokio::test] async fn drain_one_marks_delivered_on_success() { let id = Uuid::new_v4(); let row = pending_row(id); let mut after = row.clone(); after.status = "delivered".into(); after.attempts = 1; after.delivered_at = Some(Utc::now()); let db: DatabaseConnection = MockDatabase::new(DatabaseBackend::Postgres) // SELECT for candidate .append_query_results([vec![row.clone()]]) // Row returned by .update() .append_query_results([vec![after]]) .append_exec_results([MockExecResult { last_insert_id: 0, rows_affected: 1, }]) .into_connection(); let updated = FediverseOutboundService::drain_one(&db, &NoopDelivery) .await .unwrap() .expect("row processed"); assert_eq!(updated.status, "delivered"); assert_eq!(updated.attempts, 1); assert!(updated.last_error.is_none()); } #[tokio::test] async fn drain_one_marks_failed_pending_with_next_retry_at_when_below_max_attempts() { // RFC 0010: a single failure on a row with attempts < max // keeps the row `pending` but gated on `next_retry_at`. let id = Uuid::new_v4(); let row = pending_row(id); let mut after = row.clone(); after.status = "pending".into(); after.attempts = 1; after.last_error = Some("simulated transport failure".into()); after.next_retry_at = Some(Utc::now() + chrono::Duration::minutes(5)); let db: DatabaseConnection = MockDatabase::new(DatabaseBackend::Postgres) .append_query_results([vec![row.clone()]]) .append_query_results([vec![after]]) .append_exec_results([MockExecResult { last_insert_id: 0, rows_affected: 1, }]) .into_connection(); let updated = FediverseOutboundService::drain_one(&db, &ErringDelivery) .await .unwrap() .expect("row processed"); assert_eq!(updated.status, "pending"); assert!(updated.last_error.is_some()); assert!(updated.next_retry_at.is_some()); } #[tokio::test] async fn drain_one_marks_dead_when_attempts_reach_max() { let id = Uuid::new_v4(); let mut row = pending_row(id); row.attempts = 7; row.max_attempts = 8; let mut after = row.clone(); after.status = "dead".into(); after.attempts = 8; after.last_error = Some("simulated transport failure".into()); let db: DatabaseConnection = MockDatabase::new(DatabaseBackend::Postgres) .append_query_results([vec![row]]) .append_query_results([vec![after]]) .append_exec_results([MockExecResult { last_insert_id: 0, rows_affected: 1, }]) .into_connection(); let updated = FediverseOutboundService::drain_one(&db, &ErringDelivery) .await .unwrap() .unwrap(); assert_eq!(updated.status, "dead"); assert!(updated.next_retry_at.is_none()); } #[test] fn next_retry_offset_grows_then_caps() { let secs: Vec<u64> = (1..=8) .map(|i| OutboundStatus::next_retry_offset(i).as_secs()) .collect(); // Strictly increasing for the first 6 entries. assert!(secs[0] < secs[1]); assert!(secs[1] < secs[2]); assert!(secs[2] < secs[3]); assert!(secs[3] < secs[4]); assert!(secs[4] < secs[5]); // Cap at 48h afterwards. assert_eq!(secs[5], 48 * 3600); assert_eq!(secs[6], 48 * 3600); assert_eq!(secs[7], 48 * 3600); } #[tokio::test] async fn drain_one_increments_attempts() { let id = Uuid::new_v4(); let mut row = pending_row(id); row.attempts = 3; let mut after = row.clone(); after.status = "delivered".into(); after.attempts = 4; after.delivered_at = Some(Utc::now()); let db: DatabaseConnection = MockDatabase::new(DatabaseBackend::Postgres) .append_query_results([vec![row.clone()]]) .append_query_results([vec![after]]) .append_exec_results([MockExecResult { last_insert_id: 0, rows_affected: 1, }]) .into_connection(); let updated = FediverseOutboundService::drain_one(&db, &NoopDelivery) .await .unwrap() .unwrap(); assert_eq!(updated.attempts, 4); } #[tokio::test] async fn noop_delivery_returns_ok() { let row = pending_row(Uuid::new_v4()); assert!(NoopDelivery.deliver(&row).await.is_ok()); } #[tokio::test] async fn worker_can_be_built_and_dropped() { // Smoke test : building the struct and dropping it before the // loop runs must not panic. let db: DatabaseConnection = MockDatabase::new(DatabaseBackend::Postgres).into_connection(); let _worker = OutboundWorker::new(db, NoopDelivery, Duration::from_millis(10)); // Drop happens at end of scope. } }
GitRust