Skip to main content

fleetforge_runtime_api/
lib.rs

1//! FleetForge gRPC API surface definitions and server wiring.
2
3mod auth;
4mod http;
5mod transparency;
6mod transparency_writer;
7
8use std::collections::{HashMap, HashSet, VecDeque};
9use std::convert::{Infallible, TryFrom};
10use std::env;
11use std::net::SocketAddr;
12use std::sync::Arc;
13
14use ::http::{header, HeaderValue, Method};
15use anyhow::{anyhow, Context, Result};
16use auth::{AuthError, AuthHandle, Role};
17use bytes::Bytes;
18use chrono::{DateTime, TimeZone, Utc};
19use clickhouse::{Client as ClickhouseClient, Row};
20use fleetforge_changeops::{
21    BudgetSignal as GateBudgetSignal, ChangeDecisionEffect as GateEffect, ChangeDiff as GateDiff,
22    ChangeFollowupRecord, ChangeFollowupRequest as GateFollowupRequest,
23    ChangeGateDecision as GateDecision, ChangeGateEngine, ChangeGateRequest as GateRequest,
24    CoverageComponent as GateCoverageComponent, CoverageReport as GateCoverageReport,
25    DecisionScorecard as GateScorecard, EvalMetric as GateEvalMetric,
26    FollowupAction as GateFollowupAction, FollowupOutcome as GateFollowupOutcome,
27    MetricDirection as GateMetricDirection,
28};
29use fleetforge_common::{
30    labels::{labels_map_to_value, labels_value_to_map},
31    licensing::{feature_allowed, LicensedFeature},
32    prost_json::{json_to_prost_struct, json_to_prost_value, prost_struct_to_json},
33    validation::{validate_run_spec, validate_step_spec, SchemaValidationError},
34};
35use fleetforge_contracts::{
36    runtime,
37    tap::{
38        self, subscribe_run_request, EventSeverity, PolicyDecisionEffect, QueryEventSource,
39        QueryEventsRequest, QueryEventsResponse, RunEventEnvelope, RunTraceContext,
40        SubscribeRunAck, SubscribeRunOpen, SubscribeRunRequest, SubscribeRunResponse,
41    },
42    RuntimeService, RuntimeServiceClient, RuntimeServiceServer, TapService, TapServiceClient,
43    TapServiceServer,
44};
45use fleetforge_storage::{models, models::NewOutboxEvent, Storage};
46use fleetforge_telemetry::audit::Audit;
47use fleetforge_trust::{
48    Attestation, AttestationVault, InMemoryAttestationVault, ObjectStoreAttestationVault,
49    TrustSubject,
50};
51use futures::Stream;
52use http_body::Body as _;
53use hyper::service::{make_service_fn, service_fn};
54use hyper::{Body, Request as HyperRequest, Server as HyperServer};
55use serde::Deserialize;
56use serde_json::{json, Map, Number, Value};
57use std::pin::Pin;
58use tokio::net::TcpListener;
59use tokio::sync::mpsc;
60use tokio::time::{interval, sleep, Duration, MissedTickBehavior};
61use tokio_stream::wrappers::ReceiverStream;
62use tonic::transport::Server;
63use tonic::{Request, Response, Status};
64use tonic_health::pb::health_check_response::ServingStatus;
65use tonic_health::server::health_reporter;
66use tonic_reflection::server::Builder as ReflectionBuilder;
67use tower::util::BoxCloneService;
68use tower::{Service, ServiceExt};
69use tower_http::cors::CorsLayer;
70use tracing::{error, info, instrument, warn};
71use transparency::{transparency_log_from_env, TransparencyContext, TransparencyLog};
72use transparency_writer::TransparencyWriter;
73use uuid::Uuid;
74
75#[derive(Clone, Debug)]
76pub struct RuntimeMetadata {
77    pub commit_sha: String,
78    pub policy_pack_id: String,
79    pub policy_pack_digest: String,
80    pub trust_mesh_alpha: bool,
81}
82
83impl RuntimeMetadata {
84    pub fn new(
85        commit_sha: impl Into<String>,
86        policy_pack_id: impl Into<String>,
87        policy_pack_digest: impl Into<String>,
88        trust_mesh_alpha: bool,
89    ) -> Self {
90        Self {
91            commit_sha: commit_sha.into(),
92            policy_pack_id: policy_pack_id.into(),
93            policy_pack_digest: policy_pack_digest.into(),
94            trust_mesh_alpha,
95        }
96    }
97}
98
99#[derive(Clone, Copy, Debug, PartialEq, Eq)]
100enum TransparencyScope {
101    Disabled,
102    Gates,
103    Runs,
104    Artifacts,
105}
106
107impl TransparencyScope {
108    fn includes_gates(self) -> bool {
109        matches!(
110            self,
111            TransparencyScope::Gates | TransparencyScope::Runs | TransparencyScope::Artifacts
112        )
113    }
114}
115#[derive(Clone)]
116struct RuntimeApiService {
117    storage: Arc<Storage>,
118    auth: Arc<AuthHandle>,
119    audit: Arc<Audit>,
120    attestation_vault: Arc<dyn AttestationVault>,
121    changeops: ChangeGateEngine,
122    transparency: Arc<dyn TransparencyLog>,
123    transparency_writer_enabled: bool,
124    transparency_scope: TransparencyScope,
125}
126
127impl RuntimeApiService {
128    fn new(
129        storage: Arc<Storage>,
130        auth: Arc<AuthHandle>,
131        audit: Arc<Audit>,
132        attestation_vault: Arc<dyn AttestationVault>,
133        transparency: Arc<dyn TransparencyLog>,
134        transparency_writer_enabled: bool,
135        transparency_scope: TransparencyScope,
136    ) -> Self {
137        Self {
138            storage,
139            auth,
140            audit,
141            attestation_vault,
142            changeops: ChangeGateEngine::default(),
143            transparency,
144            transparency_writer_enabled,
145            transparency_scope,
146        }
147    }
148
149    async fn require_role(
150        &self,
151        metadata: &tonic::metadata::MetadataMap,
152        required: Role,
153    ) -> Result<Role, Status> {
154        self.auth
155            .require_grpc(metadata, required)
156            .await
157            .map_err(|err| match err {
158                AuthError::MissingToken | AuthError::InvalidToken => {
159                    Status::unauthenticated(err.to_string())
160                }
161                AuthError::InsufficientRole { .. }
162                | AuthError::MissingRoleClaim(_)
163                | AuthError::UnsupportedRole { .. } => Status::permission_denied(err.to_string()),
164                AuthError::NotConfigured => Status::unauthenticated(err.to_string()),
165                AuthError::Backend(_) => Status::internal(err.to_string()),
166            })
167    }
168
169    fn actor_from_metadata(metadata: &tonic::metadata::MetadataMap, role: Role) -> String {
170        metadata
171            .get("x-actor")
172            .and_then(|value| value.to_str().ok())
173            .map(|value| value.to_string())
174            .unwrap_or_else(|| format!("role:{role:?}"))
175    }
176}
177
178fn invalid_argument(message: impl Into<String>) -> Status {
179    Status::invalid_argument(message.into())
180}
181
182fn internal_error(message: impl Into<String>) -> Status {
183    Status::internal(message.into())
184}
185
186const QUERY_DEFAULT_PAGE_SIZE: u32 = 100;
187const QUERY_MAX_PAGE_SIZE: u32 = 500;
188const QUERY_FETCH_MULTIPLIER: u32 = 4;
189const TRANSPARENCY_WRITER_FLAG: &str = "FLEETFORGE_TRANSPARENCY_WRITER";
190const TRANSPARENCY_WRITER_INTERVAL_ENV: &str = "FLEETFORGE_TRANSPARENCY_WRITER_INTERVAL_SECS";
191const TRANSPARENCY_WRITER_INTERVAL_MIN_SECS: u64 = 5;
192const TRANSPARENCY_WRITER_DEFAULT_INTERVAL_SECS: u64 = 30;
193
194fn to_timestamp(time: DateTime<Utc>) -> prost_types::Timestamp {
195    prost_types::Timestamp {
196        seconds: time.timestamp(),
197        nanos: time.timestamp_subsec_nanos() as i32,
198    }
199}
200
201fn value_to_struct(value: &Value) -> Option<prost_types::Struct> {
202    if value.is_null() {
203        return None;
204    }
205    match json_to_prost_struct(value) {
206        Ok(struct_value) => Some(struct_value),
207        Err(err) => {
208            warn!(error = %err, "failed to convert JSON value to protobuf Struct");
209            None
210        }
211    }
212}
213
214impl RuntimeApiService {
215    async fn publish_scitt_entry(
216        &self,
217        change_id: &str,
218        gate_id: Uuid,
219        artifact_sha: &str,
220        attestation_refs: &[Uuid],
221        metadata: &mut Value,
222    ) -> Result<(), Status> {
223        let signer = fleetforge_trust::scitt_signer()
224            .map_err(|err| internal_error(format!("SCITT signer unavailable: {err}")))?;
225        let scitt_entry = fleetforge_trust::build_scitt_entry(
226            change_id,
227            attestation_refs,
228            artifact_sha,
229            metadata,
230            signer.as_ref(),
231        )
232        .map_err(|err| internal_error(format!("failed to build SCITT entry: {err}")))?;
233
234        let transparency_ctx = TransparencyContext {
235            change_id,
236            gate_id: &gate_id,
237            artifact_sha256: artifact_sha,
238            attestation_ids: attestation_refs,
239        };
240        let record = self
241            .transparency
242            .append(&scitt_entry, &transparency_ctx)
243            .await
244            .map_err(|err| internal_error(format!("failed to publish SCITT entry: {err}")))?;
245
246        let scitt_metadata = json!({
247            "id": record.entry_id,
248            "sha256": hex::encode(record.artifact.sha256),
249            "media_type": record.artifact.media_type.clone(),
250            "size": record.artifact.size,
251            "attestation_ids": attestation_refs.iter().map(|id| id.to_string()).collect::<Vec<_>>(),
252            "transparency": record.receipt,
253        });
254
255        if let Value::Object(mut meta_obj) = metadata.take() {
256            meta_obj.insert("scitt_entry".to_string(), scitt_metadata);
257            *metadata = Value::Object(meta_obj);
258        } else {
259            *metadata = json!({ "scitt_entry": scitt_metadata });
260        }
261        Ok(())
262    }
263}
264
265fn struct_to_value(data: Option<prost_types::Struct>) -> Value {
266    data.as_ref()
267        .map(prost_struct_to_json)
268        .unwrap_or(Value::Null)
269}
270
271fn attestation_to_proto(att: Attestation) -> Result<runtime::Attestation, Status> {
272    let subject = att
273        .subject
274        .as_ref()
275        .map(attestation_subject_to_proto)
276        .transpose()?;
277
278    let claims_value = Value::Object(att.claims.into_iter().collect());
279    let claims_struct = json_to_prost_struct(&claims_value)
280        .map_err(|err| internal_error(format!("failed to serialise attestation claims: {err}")))?;
281
282    Ok(runtime::Attestation {
283        attestation_id: att.id.to_string(),
284        issued_at: Some(to_timestamp(att.issued_at)),
285        subject,
286        artifact_hash: att.artifact_hash.map(|value| value.into()),
287        claims: Some(claims_struct),
288        signature: att.signature.map(|value| value.into()),
289    })
290}
291
292fn attestation_subject_to_proto(
293    subject: &TrustSubject,
294) -> Result<runtime::AttestationSubject, Status> {
295    use runtime::attestation_subject::Kind;
296    let kind = match subject {
297        TrustSubject::Run { run_id } => Kind::Run(runtime::AttestationRunSubject {
298            run_id: run_id.to_string(),
299        }),
300        TrustSubject::Step { run_id, step_id } => Kind::Step(runtime::AttestationStepSubject {
301            run_id: run_id.to_string(),
302            step_id: step_id.to_string(),
303        }),
304        TrustSubject::Tool { name } => {
305            Kind::Tool(runtime::AttestationToolSubject { name: name.clone() })
306        }
307        TrustSubject::Artifact { uri } => {
308            Kind::Artifact(runtime::AttestationArtifactSubject { uri: uri.clone() })
309        }
310        TrustSubject::CapabilityToken { token_id } => {
311            Kind::CapabilityToken(runtime::AttestationCapabilitySubject {
312                token_id: token_id.to_string(),
313            })
314        }
315        TrustSubject::Custom { label } => Kind::Custom(runtime::AttestationCustomSubject {
316            label: label.clone(),
317        }),
318    };
319    Ok(runtime::AttestationSubject { kind: Some(kind) })
320}
321
322fn gate_effect_to_proto(effect: GateEffect) -> runtime::ChangeDecisionEffect {
323    match effect {
324        GateEffect::Allow => runtime::ChangeDecisionEffect::Allow,
325        GateEffect::FollowUp => runtime::ChangeDecisionEffect::FollowUp,
326        GateEffect::Deny => runtime::ChangeDecisionEffect::Deny,
327    }
328}
329
330fn gate_effect_from_str(value: &str) -> GateEffect {
331    match value {
332        "allow" => GateEffect::Allow,
333        "deny" => GateEffect::Deny,
334        "follow_up" => GateEffect::FollowUp,
335        other => {
336            warn!(effect = %other, "unknown change gate effect stored; defaulting to follow_up");
337            GateEffect::FollowUp
338        }
339    }
340}
341
342fn gate_followup_action_to_proto(action: &GateFollowupAction) -> runtime::FollowupAction {
343    runtime::FollowupAction {
344        slug: action.slug.clone(),
345        description: action.description.clone(),
346        recommended: action.recommended,
347    }
348}
349
350fn gate_scorecard_to_proto(scorecard: &GateScorecard) -> Option<runtime::DecisionScorecard> {
351    Some(runtime::DecisionScorecard {
352        novelty: Some(runtime::NoveltySummary {
353            max_novelty: scorecard.novelty.max_novelty,
354            avg_novelty: scorecard.novelty.avg_novelty,
355            high_risk_paths: scorecard.novelty.high_risk_paths.clone(),
356        }),
357        coverage: Some(runtime::CoverageSummary {
358            overall_ratio: scorecard.coverage.overall_ratio,
359            components_below_threshold: scorecard.coverage.components_below_threshold.clone(),
360        }),
361        evals: Some(runtime::EvalSummary {
362            failing_metrics: scorecard.evals.failing_metrics.clone(),
363            attention_metrics: scorecard.evals.attention_metrics.clone(),
364            score_per_token: scorecard.evals.score_per_token,
365        }),
366        budgets: Some(runtime::BudgetSummary {
367            breaches: scorecard.budgets.breaches.clone(),
368            near_limits: scorecard.budgets.near_limits.clone(),
369        }),
370        telemetry: value_to_struct(&scorecard.telemetry),
371    })
372}
373
374fn proto_followup_outcome(outcome: runtime::FollowupOutcome) -> GateFollowupOutcome {
375    match outcome {
376        runtime::FollowupOutcome::Approved => GateFollowupOutcome::Approved,
377        runtime::FollowupOutcome::NeedsChanges => GateFollowupOutcome::NeedsChanges,
378        runtime::FollowupOutcome::Deferred => GateFollowupOutcome::Deferred,
379        runtime::FollowupOutcome::Unspecified => GateFollowupOutcome::Approved,
380    }
381}
382
383fn followup_outcome_to_proto(outcome: GateFollowupOutcome) -> runtime::FollowupOutcome {
384    match outcome {
385        GateFollowupOutcome::Approved => runtime::FollowupOutcome::Approved,
386        GateFollowupOutcome::NeedsChanges => runtime::FollowupOutcome::NeedsChanges,
387        GateFollowupOutcome::Deferred => runtime::FollowupOutcome::Deferred,
388    }
389}
390
391fn parse_reasons(value: &Value) -> Vec<String> {
392    serde_json::from_value::<Vec<String>>(value.clone()).unwrap_or_default()
393}
394
395fn parse_followup_actions(value: &Value) -> Vec<GateFollowupAction> {
396    serde_json::from_value::<Vec<GateFollowupAction>>(value.clone()).unwrap_or_default()
397}
398
399fn parse_scorecard(value: &Value) -> GateScorecard {
400    serde_json::from_value::<GateScorecard>(value.clone()).unwrap_or_default()
401}
402
403fn gate_followup_record_to_proto(
404    record: &models::ChangeGateFollowup,
405) -> runtime::ChangeGateFollowup {
406    let outcome =
407        GateFollowupOutcome::from_str(&record.outcome).unwrap_or(GateFollowupOutcome::Approved);
408    let details_struct = value_to_struct(&record.details);
409
410    runtime::ChangeGateFollowup {
411        followup_id: record.followup_id.to_string(),
412        gate_id: record.gate_id.to_string(),
413        actor: record.actor.clone(),
414        outcome: followup_outcome_to_proto(outcome) as i32,
415        note: record.note.clone(),
416        details: details_struct,
417        recorded_at: Some(to_timestamp(record.recorded_at)),
418    }
419}
420
421fn build_proto_change_gate(
422    gate_id: Uuid,
423    change_id: &str,
424    revision: Option<&str>,
425    decision: &GateDecision,
426    telemetry: &Value,
427    acknowledgements: &[models::ChangeGateFollowup],
428) -> Result<runtime::ChangeGateDecision, Status> {
429    let effect = gate_effect_to_proto(decision.effect) as i32;
430    let scorecard = gate_scorecard_to_proto(&decision.scorecard);
431    let metadata_struct = value_to_struct(&decision.metadata);
432    let telemetry_struct = value_to_struct(telemetry);
433    let ack_proto = acknowledgements
434        .iter()
435        .map(gate_followup_record_to_proto)
436        .collect();
437
438    Ok(runtime::ChangeGateDecision {
439        gate_id: gate_id.to_string(),
440        change_id: change_id.to_string(),
441        revision: revision.unwrap_or_default().to_string(),
442        effect,
443        reasons: decision.reasons.clone(),
444        followups: decision
445            .followups
446            .iter()
447            .map(gate_followup_action_to_proto)
448            .collect(),
449        scorecard,
450        decided_at: Some(to_timestamp(decision.decided_at)),
451        metadata: metadata_struct,
452        telemetry: telemetry_struct,
453        acknowledgements: ack_proto,
454    })
455}
456
457fn attestation_ids_from_value(value: &Value) -> Vec<Uuid> {
458    fn recurse(value: &Value, acc: &mut HashSet<Uuid>) {
459        match value {
460            Value::Object(map) => {
461                if let Some(ids) = map.get("attestation_ids").and_then(Value::as_array) {
462                    for id in ids {
463                        if let Some(s) = id.as_str() {
464                            if let Ok(uuid) = Uuid::parse_str(s) {
465                                acc.insert(uuid);
466                            }
467                        }
468                    }
469                }
470                for v in map.values() {
471                    recurse(v, acc);
472                }
473            }
474            Value::Array(items) => {
475                for item in items {
476                    recurse(item, acc);
477                }
478            }
479            _ => {}
480        }
481    }
482    let mut set = HashSet::new();
483    recurse(value, &mut set);
484    set.into_iter().collect()
485}
486
487#[derive(Clone)]
488struct TapApiService {
489    storage: Arc<Storage>,
490    auth: Arc<AuthHandle>,
491    clickhouse: Option<ClickhouseEventQuery>,
492}
493
494impl TapApiService {
495    fn new(storage: Arc<Storage>, auth: Arc<AuthHandle>) -> Self {
496        let clickhouse = ClickhouseEventQuery::from_env().ok();
497        Self {
498            storage,
499            auth,
500            clickhouse,
501        }
502    }
503
504    async fn require_role(
505        &self,
506        metadata: &tonic::metadata::MetadataMap,
507        required: Role,
508    ) -> Result<Role, Status> {
509        self.auth
510            .require_grpc(metadata, required)
511            .await
512            .map_err(|err| match err {
513                AuthError::MissingToken | AuthError::InvalidToken => {
514                    Status::unauthenticated(err.to_string())
515                }
516                AuthError::InsufficientRole { .. }
517                | AuthError::MissingRoleClaim(_)
518                | AuthError::UnsupportedRole { .. } => Status::permission_denied(err.to_string()),
519                AuthError::NotConfigured => Status::unauthenticated(err.to_string()),
520                AuthError::Backend(_) => Status::internal(err.to_string()),
521            })
522    }
523}
524
525#[tonic::async_trait]
526impl TapService for TapApiService {
527    type SubscribeRunStream =
528        Pin<Box<dyn Stream<Item = Result<SubscribeRunResponse, Status>> + Send>>;
529
530    async fn subscribe_run(
531        &self,
532        request: Request<tonic::Streaming<SubscribeRunRequest>>,
533    ) -> Result<Response<Self::SubscribeRunStream>, Status> {
534        let metadata = request.metadata().clone();
535        self.require_role(&metadata, Role::Reader).await?;
536
537        let mut inbound = request.into_inner();
538        let first = inbound
539            .message()
540            .await
541            .map_err(|status| internal_error(status.to_string()))?
542            .ok_or_else(|| invalid_argument("missing subscribe request"))?;
543
544        let open = match first.request {
545            Some(subscribe_run_request::Request::Open(open)) => open,
546            Some(subscribe_run_request::Request::Ack(_)) => {
547                return Err(invalid_argument(
548                    "first SubscribeRun message must be `open`",
549                ))
550            }
551            None => return Err(invalid_argument("first SubscribeRun message must be set")),
552        };
553
554        let mut config = SubscriptionConfig::try_from_open(open)?;
555        if let Some(cursor) = config.cursor.clone() {
556            if config.since_offset == 0 {
557                match self.storage.fetch_tap_offset(config.run_id, &cursor).await {
558                    Ok(Some(stored)) => {
559                        config.since_offset = stored;
560                    }
561                    Ok(None) => {}
562                    Err(err) => {
563                        error!(
564                            run = %config.run_id,
565                            cursor = %cursor,
566                            error = %err,
567                            "failed to load stored tap cursor"
568                        );
569                        return Err(internal_error("failed to load stored tap cursor"));
570                    }
571                }
572            }
573        }
574        let (tx, rx) =
575            mpsc::channel::<Result<SubscribeRunResponse, Status>>(config.max_inflight * 2);
576
577        let storage = Arc::clone(&self.storage);
578        tokio::spawn(async move {
579            if let Err(err) = run_subscription(storage, config, inbound, tx.clone()).await {
580                let _ = tx.send(Err(err)).await;
581            }
582        });
583
584        Ok(Response::new(
585            Box::pin(ReceiverStream::new(rx)) as Self::SubscribeRunStream
586        ))
587    }
588
589    async fn query_events(
590        &self,
591        request: Request<QueryEventsRequest>,
592    ) -> Result<Response<QueryEventsResponse>, Status> {
593        let metadata = request.metadata().clone();
594        self.require_role(&metadata, Role::Reader).await?;
595
596        let clickhouse = self.clickhouse.as_ref().ok_or_else(|| {
597            Status::failed_precondition("QueryEvents requires ClickHouse configuration")
598        })?;
599
600        let inner = request.into_inner();
601        let filters = QueryFiltersOwned::from_request(&inner)?;
602
603        let requested_page = if inner.page_size == 0 {
604            QUERY_DEFAULT_PAGE_SIZE
605        } else {
606            inner.page_size
607        };
608        let page_size = requested_page.clamp(1, QUERY_MAX_PAGE_SIZE);
609        let offset = parse_page_token(inner.page_token.as_str())?;
610
611        let QueryBatch {
612            events,
613            next_offset,
614        } = clickhouse
615            .fetch(&filters, offset, page_size)
616            .await
617            .map_err(|err| {
618                error!(error = %err, "failed to query ClickHouse events");
619                internal_error("failed to query ClickHouse events")
620            })?;
621
622        let next_page_token = next_offset
623            .map(|value| value.to_string())
624            .unwrap_or_default();
625
626        Ok(Response::new(QueryEventsResponse {
627            events,
628            next_page_token,
629        }))
630    }
631}
632
633struct SubscriptionConfig {
634    run_id: Uuid,
635    step_ids: Option<HashSet<Uuid>>,
636    kinds: Option<HashSet<String>>,
637    severities: Option<HashSet<EventSeverity>>,
638    tags: Option<HashSet<String>>,
639    cursor: Option<String>,
640    since_offset: i64,
641    batch_size: i64,
642    max_inflight: usize,
643    heartbeat: Duration,
644}
645
646impl SubscriptionConfig {
647    fn try_from_open(open: SubscribeRunOpen) -> Result<Self, Status> {
648        let SubscribeRunOpen {
649            run_ids,
650            step_ids,
651            kinds,
652            severities,
653            tags,
654            since_offset,
655            batch_size,
656            max_inflight,
657            heartbeat_interval_ms,
658            cursor,
659            ..
660        } = open;
661
662        if run_ids.is_empty() {
663            return Err(invalid_argument(
664                "subscribe_run requires at least one run_id",
665            ));
666        }
667        if run_ids.len() > 1 {
668            return Err(invalid_argument(
669                "subscribe_run currently supports a single run_id",
670            ));
671        }
672        let run_id = Uuid::parse_str(&run_ids[0])
673            .map_err(|err| invalid_argument(format!("invalid run_id '{}' : {err}", run_ids[0])))?;
674
675        let step_ids = if step_ids.is_empty() {
676            None
677        } else {
678            let mut ids = HashSet::with_capacity(step_ids.len());
679            for raw in step_ids {
680                let parsed = Uuid::parse_str(&raw)
681                    .map_err(|err| invalid_argument(format!("invalid step_id '{raw}': {err}")))?;
682                ids.insert(parsed);
683            }
684            Some(ids)
685        };
686
687        let kinds = if kinds.is_empty() {
688            None
689        } else {
690            Some(
691                kinds
692                    .into_iter()
693                    .map(|kind| kind.to_ascii_lowercase())
694                    .collect::<HashSet<_>>(),
695            )
696        };
697
698        let severities = if severities.is_empty() {
699            None
700        } else {
701            let mut set = HashSet::with_capacity(severities.len());
702            for raw in severities {
703                let sev = EventSeverity::try_from(raw)
704                    .map_err(|_| invalid_argument(format!("unknown severity enum value {raw}")))?;
705                set.insert(sev);
706            }
707            Some(set)
708        };
709
710        let tags = if tags.is_empty() {
711            None
712        } else {
713            Some(
714                tags.into_iter()
715                    .map(|tag| tag.to_ascii_lowercase())
716                    .collect::<HashSet<_>>(),
717            )
718        };
719
720        if since_offset < 0 {
721            return Err(invalid_argument("since_offset must be >= 0"));
722        }
723
724        let batch_raw = if batch_size == 0 { 256 } else { batch_size };
725        let batch_size = batch_raw.max(1).min(2048) as i64;
726
727        let inflight_raw = if max_inflight == 0 { 128 } else { max_inflight };
728        let max_inflight = inflight_raw.max(1).min(4096) as usize;
729
730        let heartbeat_ms = if heartbeat_interval_ms == 0 {
731            1000u64
732        } else {
733            heartbeat_interval_ms as u64
734        }
735        .clamp(250, 60_000);
736
737        let cursor = {
738            let trimmed = cursor.trim();
739            if trimmed.is_empty() {
740                None
741            } else {
742                Some(trimmed.to_string())
743            }
744        };
745
746        Ok(Self {
747            run_id,
748            step_ids,
749            kinds,
750            severities,
751            tags,
752            cursor,
753            since_offset,
754            batch_size,
755            max_inflight,
756            heartbeat: Duration::from_millis(heartbeat_ms),
757        })
758    }
759}
760
761async fn run_subscription(
762    storage: Arc<Storage>,
763    config: SubscriptionConfig,
764    mut inbound: tonic::Streaming<SubscribeRunRequest>,
765    tx: mpsc::Sender<Result<SubscribeRunResponse, Status>>,
766) -> Result<(), Status> {
767    let mut last_seen = config.since_offset;
768    let mut last_ack = config.since_offset;
769    let mut inflight: VecDeque<i64> = VecDeque::new();
770    let mut ticker = interval(config.heartbeat);
771    ticker.set_missed_tick_behavior(MissedTickBehavior::Delay);
772
773    loop {
774        while inflight.len() < config.max_inflight {
775            let events = storage
776                .fetch_outbox_for_run_since(config.run_id, last_seen, config.batch_size)
777                .await
778                .map_err(|err| internal_error(format!("failed to fetch outbox events: {err}")))?;
779
780            if events.is_empty() {
781                break;
782            }
783
784            last_seen = events.last().map(|event| event.id).unwrap_or(last_seen);
785
786            for event in events {
787                last_seen = event.id;
788                let severity = derive_severity(&event.kind, &event.payload);
789                let tags = derive_tags(&event.kind, event.step_id, &event.payload, severity);
790                if !passes_filters(&config, &event, severity, &tags) {
791                    continue;
792                }
793
794                let response = build_response(&config, &event, severity, &tags, last_seen)?;
795                if tx.send(Ok(response)).await.is_err() {
796                    return Ok(());
797                }
798                inflight.push_back(event.id);
799            }
800        }
801
802        tokio::select! {
803            maybe_req = inbound.message() => {
804                match maybe_req {
805                    Ok(Some(SubscribeRunRequest { request: Some(subscribe_run_request::Request::Ack(ack)) })) => {
806                        handle_ack(&mut inflight, &mut last_ack, ack.offset);
807                        let ack_cursor = if !ack.cursor.trim().is_empty() {
808                            Some(ack.cursor)
809                        } else {
810                            config.cursor.clone()
811                        };
812                        if let Some(cursor_id) = ack_cursor {
813                            if ack.offset < 0 {
814                                return Err(invalid_argument("ack offset must be >= 0"));
815                            }
816                            if let Err(err) = storage
817                                .upsert_tap_offset(config.run_id, &cursor_id, ack.offset)
818                                .await
819                            {
820                                error!(
821                                    run = %config.run_id,
822                                    cursor = %cursor_id,
823                                    error = %err,
824                                    "failed to persist tap cursor offset"
825                                );
826                                return Err(internal_error("failed to persist tap cursor offset"));
827                            }
828                        }
829                    }
830                    Ok(Some(SubscribeRunRequest { request: Some(subscribe_run_request::Request::Open(_)) })) => {
831                        return Err(invalid_argument("subscribe_run received multiple open messages"));
832                    }
833                    Ok(Some(SubscribeRunRequest { request: None })) => {}
834                    Ok(None) => {
835                        return Ok(());
836                    }
837                    Err(status) => {
838                        return Err(internal_error(status.to_string()));
839                    }
840                }
841            }
842            _ = ticker.tick() => {
843                let heartbeat = SubscribeRunResponse {
844                    offset: last_ack,
845                    watermark: last_seen,
846                    heartbeat: true,
847                    event: None,
848                };
849                if tx.send(Ok(heartbeat)).await.is_err() {
850                    return Ok(());
851                }
852            }
853        }
854    }
855}
856
857#[derive(Debug, Clone)]
858struct QueryFiltersOwned {
859    source: QueryEventSource,
860    run_ids: Vec<Uuid>,
861    step_ids: Vec<Uuid>,
862    workspace_ids: Vec<String>,
863    kinds: Vec<String>,
864    policy_effects: Vec<PolicyDecisionEffect>,
865    severities: Vec<EventSeverity>,
866    start_time: Option<DateTime<Utc>>,
867    end_time: Option<DateTime<Utc>>,
868    ascending: bool,
869}
870
871impl QueryFiltersOwned {
872    fn from_request(request: &QueryEventsRequest) -> Result<Self, Status> {
873        let mut run_ids = Vec::with_capacity(request.run_ids.len());
874        for raw in &request.run_ids {
875            let parsed = Uuid::parse_str(raw)
876                .map_err(|err| invalid_argument(format!("invalid run_id '{raw}': {err}")))?;
877            run_ids.push(parsed);
878        }
879
880        let mut step_ids = Vec::with_capacity(request.step_ids.len());
881        for raw in &request.step_ids {
882            let parsed = Uuid::parse_str(raw)
883                .map_err(|err| invalid_argument(format!("invalid step_id '{raw}': {err}")))?;
884            step_ids.push(parsed);
885        }
886
887        let source = QueryEventSource::try_from(request.source).unwrap_or(QueryEventSource::Run);
888
889        let start_time = request
890            .start_time
891            .as_ref()
892            .map(timestamp_to_datetime)
893            .transpose()?;
894        let end_time = request
895            .end_time
896            .as_ref()
897            .map(timestamp_to_datetime)
898            .transpose()?;
899
900        let kinds = request
901            .kinds
902            .iter()
903            .filter(|kind| !kind.is_empty())
904            .map(|kind| kind.to_ascii_lowercase())
905            .collect::<Vec<_>>();
906
907        let workspace_ids = request
908            .workspace_ids
909            .iter()
910            .filter(|id| !id.is_empty())
911            .map(|id| id.to_ascii_lowercase())
912            .collect::<Vec<_>>();
913
914        let mut policy_effects = Vec::with_capacity(request.policy_effects.len());
915        for raw in &request.policy_effects {
916            if let Ok(effect) = PolicyDecisionEffect::try_from(*raw) {
917                if effect != PolicyDecisionEffect::Unspecified {
918                    policy_effects.push(effect);
919                }
920            }
921        }
922
923        let mut severities = Vec::with_capacity(request.severities.len());
924        for raw in &request.severities {
925            if let Ok(severity) = EventSeverity::try_from(*raw) {
926                if severity != EventSeverity::Unspecified {
927                    severities.push(severity);
928                }
929            }
930        }
931
932        Ok(Self {
933            source,
934            run_ids,
935            step_ids,
936            workspace_ids,
937            kinds,
938            policy_effects,
939            severities,
940            start_time,
941            end_time,
942            ascending: request.ascending,
943        })
944    }
945}
946
947struct QueryBatch {
948    events: Vec<RunEventEnvelope>,
949    next_offset: Option<u64>,
950}
951
952#[derive(Clone)]
953struct ClickhouseEventQuery {
954    client: ClickhouseClient,
955    run_table: String,
956    step_table: String,
957}
958
959#[derive(Debug, Deserialize, Row)]
960struct RunEventRow {
961    run_id: Uuid,
962    created_at: DateTime<Utc>,
963    status: String,
964    attributes: Value,
965}
966
967#[derive(Debug, Deserialize, Row)]
968struct StepEventRow {
969    run_id: Uuid,
970    step_id: Uuid,
971    status: String,
972    attempt: i32,
973    recorded_at: DateTime<Utc>,
974    attributes: Value,
975}
976
977impl ClickhouseEventQuery {
978    fn from_env() -> Result<Self> {
979        let url = env::var("CLICKHOUSE_DSN")
980            .context("CLICKHOUSE_DSN is not configured for Tap QueryEvents")?;
981        let database = env::var("CLICKHOUSE_DATABASE").unwrap_or_else(|_| "telemetry".to_string());
982        let run_table =
983            env::var("CLICKHOUSE_RUN_TABLE").unwrap_or_else(|_| "run_events".to_string());
984        let step_table =
985            env::var("CLICKHOUSE_STEP_TABLE").unwrap_or_else(|_| "step_events".to_string());
986
987        let client = ClickhouseClient::default()
988            .with_url(url)
989            .with_database(database);
990
991        Ok(Self {
992            client,
993            run_table,
994            step_table,
995        })
996    }
997
998    async fn fetch(
999        &self,
1000        filters: &QueryFiltersOwned,
1001        offset: u64,
1002        limit: u32,
1003    ) -> Result<QueryBatch> {
1004        match filters.source {
1005            QueryEventSource::Step => self.query_step_events(filters, offset, limit).await,
1006            _ => self.query_run_events(filters, offset, limit).await,
1007        }
1008    }
1009
1010    async fn query_run_events(
1011        &self,
1012        filters: &QueryFiltersOwned,
1013        offset: u64,
1014        limit: u32,
1015    ) -> Result<QueryBatch> {
1016        let chunk = std::cmp::max(limit, 1)
1017            .saturating_mul(QUERY_FETCH_MULTIPLIER)
1018            .max(limit.max(1));
1019        let mut collected = Vec::with_capacity(limit as usize);
1020        let mut scan_offset = offset;
1021        let mut next_offset = None;
1022
1023        while collected.len() < limit as usize {
1024            let rows = self.fetch_run_rows(filters, scan_offset, chunk).await?;
1025            if rows.is_empty() {
1026                break;
1027            }
1028
1029            let mut consumed = 0usize;
1030            for row in rows {
1031                let raw_offset = scan_offset + consumed as u64;
1032                consumed += 1;
1033                let payload = normalize_payload(row.attributes);
1034                let kind = format!("run_{}", row.status.to_ascii_lowercase());
1035                let severity = derive_severity(&kind, &payload);
1036                if !matches_filters(filters, &payload, &kind, severity, None) {
1037                    continue;
1038                }
1039                let envelope = make_envelope(
1040                    row.run_id,
1041                    None,
1042                    row.created_at,
1043                    &payload,
1044                    &kind,
1045                    severity,
1046                    raw_offset,
1047                )?;
1048                collected.push(envelope);
1049                if collected.len() >= limit as usize {
1050                    next_offset = Some(raw_offset.saturating_add(1));
1051                    break;
1052                }
1053            }
1054
1055            scan_offset = scan_offset.saturating_add(consumed as u64);
1056            if consumed < chunk as usize {
1057                break;
1058            }
1059        }
1060
1061        Ok(QueryBatch {
1062            events: collected,
1063            next_offset,
1064        })
1065    }
1066
1067    async fn query_step_events(
1068        &self,
1069        filters: &QueryFiltersOwned,
1070        offset: u64,
1071        limit: u32,
1072    ) -> Result<QueryBatch> {
1073        let chunk = std::cmp::max(limit, 1)
1074            .saturating_mul(QUERY_FETCH_MULTIPLIER)
1075            .max(limit.max(1));
1076        let mut collected = Vec::with_capacity(limit as usize);
1077        let mut scan_offset = offset;
1078        let mut next_offset = None;
1079
1080        while collected.len() < limit as usize {
1081            let rows = self.fetch_step_rows(filters, scan_offset, chunk).await?;
1082            if rows.is_empty() {
1083                break;
1084            }
1085
1086            let mut consumed = 0usize;
1087            for row in rows {
1088                let raw_offset = scan_offset + consumed as u64;
1089                consumed += 1;
1090                let payload = normalize_payload(row.attributes);
1091                let kind = format!("step_{}", row.status.to_ascii_lowercase());
1092                let severity = derive_severity(&kind, &payload);
1093                if !matches_filters(filters, &payload, &kind, severity, Some(row.step_id)) {
1094                    continue;
1095                }
1096                let envelope = make_envelope(
1097                    row.run_id,
1098                    Some(row.step_id),
1099                    row.recorded_at,
1100                    &payload,
1101                    &kind,
1102                    severity,
1103                    raw_offset,
1104                )?;
1105                collected.push(envelope);
1106                if collected.len() >= limit as usize {
1107                    next_offset = Some(raw_offset.saturating_add(1));
1108                    break;
1109                }
1110            }
1111
1112            scan_offset = scan_offset.saturating_add(consumed as u64);
1113            if consumed < chunk as usize {
1114                break;
1115            }
1116        }
1117
1118        Ok(QueryBatch {
1119            events: collected,
1120            next_offset,
1121        })
1122    }
1123
1124    async fn fetch_run_rows(
1125        &self,
1126        filters: &QueryFiltersOwned,
1127        offset: u64,
1128        limit: u32,
1129    ) -> Result<Vec<RunEventRow>> {
1130        let mut where_clauses = Vec::new();
1131        if !filters.run_ids.is_empty() {
1132            where_clauses.push(format!(
1133                "run_id IN ({})",
1134                format_uuid_list(&filters.run_ids)
1135            ));
1136        }
1137        if let Some(start) = filters.start_time {
1138            where_clauses.push(format!(
1139                "created_at >= toDateTime64('{}', 3, 'UTC')",
1140                start.format("%Y-%m-%d %H:%M:%S%.3f")
1141            ));
1142        }
1143        if let Some(end) = filters.end_time {
1144            where_clauses.push(format!(
1145                "created_at <= toDateTime64('{}', 3, 'UTC')",
1146                end.format("%Y-%m-%d %H:%M:%S%.3f")
1147            ));
1148        }
1149
1150        let mut sql = format!(
1151            "SELECT run_id, created_at, status, attributes FROM {}",
1152            self.run_table
1153        );
1154        if !where_clauses.is_empty() {
1155            sql.push_str(" WHERE ");
1156            sql.push_str(&where_clauses.join(" AND "));
1157        }
1158        sql.push_str(" ORDER BY created_at ");
1159        sql.push_str(if filters.ascending { "ASC" } else { "DESC" });
1160        sql.push_str(", run_id ");
1161        sql.push_str(if filters.ascending { "ASC" } else { "DESC" });
1162        sql.push_str(&format!(" LIMIT {} OFFSET {}", limit, offset));
1163
1164        let rows = self
1165            .client
1166            .query(&sql)
1167            .fetch_all::<RunEventRow>()
1168            .await
1169            .context("failed to fetch run events from ClickHouse")?;
1170        Ok(rows)
1171    }
1172
1173    async fn fetch_step_rows(
1174        &self,
1175        filters: &QueryFiltersOwned,
1176        offset: u64,
1177        limit: u32,
1178    ) -> Result<Vec<StepEventRow>> {
1179        let mut where_clauses = Vec::new();
1180        if !filters.run_ids.is_empty() {
1181            where_clauses.push(format!(
1182                "run_id IN ({})",
1183                format_uuid_list(&filters.run_ids)
1184            ));
1185        }
1186        if !filters.step_ids.is_empty() {
1187            where_clauses.push(format!(
1188                "step_id IN ({})",
1189                format_uuid_list(&filters.step_ids)
1190            ));
1191        }
1192
1193        let time_expr = "coalesce(finished_at, started_at)";
1194        if let Some(start) = filters.start_time {
1195            where_clauses.push(format!(
1196                "{time_expr} >= toDateTime64('{}', 3, 'UTC')",
1197                start.format("%Y-%m-%d %H:%M:%S%.3f")
1198            ));
1199        }
1200        if let Some(end) = filters.end_time {
1201            where_clauses.push(format!(
1202                "{time_expr} <= toDateTime64('{}', 3, 'UTC')",
1203                end.format("%Y-%m-%d %H:%M:%S%.3f")
1204            ));
1205        }
1206
1207        let mut sql = format!(
1208            "SELECT run_id, step_id, status, attempt, {time_expr} AS recorded_at, attributes FROM {}",
1209            self.step_table
1210        );
1211        if !where_clauses.is_empty() {
1212            sql.push_str(" WHERE ");
1213            sql.push_str(&where_clauses.join(" AND "));
1214        }
1215        sql.push_str(&format!(" ORDER BY {time_expr} "));
1216        sql.push_str(if filters.ascending { "ASC" } else { "DESC" });
1217        sql.push_str(", step_id ");
1218        sql.push_str(if filters.ascending { "ASC" } else { "DESC" });
1219        sql.push_str(&format!(" LIMIT {} OFFSET {}", limit, offset));
1220
1221        let rows = self
1222            .client
1223            .query(&sql)
1224            .fetch_all::<StepEventRow>()
1225            .await
1226            .context("failed to fetch step events from ClickHouse")?;
1227        Ok(rows)
1228    }
1229}
1230
1231fn parse_page_token(raw: &str) -> Result<u64, Status> {
1232    if raw.trim().is_empty() {
1233        return Ok(0);
1234    }
1235    raw.parse::<u64>()
1236        .map_err(|_| invalid_argument("page_token must be a positive integer"))
1237}
1238
1239fn timestamp_to_datetime(ts: &prost_types::Timestamp) -> Result<DateTime<Utc>, Status> {
1240    Utc.timestamp_opt(ts.seconds, ts.nanos as u32)
1241        .single()
1242        .ok_or_else(|| invalid_argument("timestamp out of range"))
1243}
1244
1245fn normalize_payload(value: Value) -> Value {
1246    match value {
1247        Value::Object(_) => value,
1248        Value::Null => Value::Object(Map::new()),
1249        other => {
1250            let mut map = Map::new();
1251            map.insert("value".to_string(), other);
1252            Value::Object(map)
1253        }
1254    }
1255}
1256
1257fn matches_filters(
1258    filters: &QueryFiltersOwned,
1259    payload: &Value,
1260    kind: &str,
1261    severity: EventSeverity,
1262    step_id: Option<Uuid>,
1263) -> bool {
1264    if !filters.kinds.is_empty() {
1265        let kind_value = kind.to_ascii_lowercase();
1266        if !filters.kinds.iter().any(|expected| expected == &kind_value) {
1267            return false;
1268        }
1269    }
1270
1271    if !filters.workspace_ids.is_empty() {
1272        let workspace = json_get_string(payload, &["trace", "run", "workspace_id"])
1273            .map(|value| value.to_ascii_lowercase());
1274        let Some(workspace) = workspace else {
1275            return false;
1276        };
1277        if !filters
1278            .workspace_ids
1279            .iter()
1280            .any(|expected| expected == &workspace)
1281        {
1282            return false;
1283        }
1284    }
1285
1286    if !filters.policy_effects.is_empty() {
1287        let effect = payload
1288            .get("effect")
1289            .and_then(Value::as_str)
1290            .map(|value| value.to_ascii_lowercase());
1291        if let Some(effect) = effect {
1292            if !filters
1293                .policy_effects
1294                .iter()
1295                .any(|expected| policy_effect_label(*expected) == effect)
1296            {
1297                return false;
1298            }
1299        } else {
1300            return false;
1301        }
1302    }
1303
1304    if !filters.severities.is_empty() && !filters.severities.contains(&severity) {
1305        return false;
1306    }
1307
1308    if filters.source == QueryEventSource::Step && !filters.step_ids.is_empty() {
1309        if let Some(step_id) = step_id {
1310            if !filters.step_ids.contains(&step_id) {
1311                return false;
1312            }
1313        } else {
1314            return false;
1315        }
1316    }
1317
1318    true
1319}
1320
1321fn policy_effect_label(effect: PolicyDecisionEffect) -> &'static str {
1322    match effect {
1323        PolicyDecisionEffect::Allow => "allow",
1324        PolicyDecisionEffect::Deny => "deny",
1325        PolicyDecisionEffect::Escalate => "escalate",
1326        PolicyDecisionEffect::Modify => "modify",
1327        _ => "",
1328    }
1329}
1330
1331fn json_get_string<'a>(value: &'a Value, path: &[&str]) -> Option<&'a str> {
1332    let mut current = value;
1333    for segment in path {
1334        current = current.get(*segment)?;
1335    }
1336    current.as_str()
1337}
1338
1339fn format_uuid_list(values: &[Uuid]) -> String {
1340    values
1341        .iter()
1342        .map(|value| format!("'{}'", value))
1343        .collect::<Vec<_>>()
1344        .join(", ")
1345}
1346
1347fn make_envelope(
1348    run_id: Uuid,
1349    step_id: Option<Uuid>,
1350    recorded_at: DateTime<Utc>,
1351    payload: &Value,
1352    kind: &str,
1353    severity: EventSeverity,
1354    raw_offset: u64,
1355) -> Result<RunEventEnvelope> {
1356    let trace = build_trace_context(run_id, step_id, payload);
1357    let tags = derive_tags(kind, step_id, payload, severity);
1358    let raw_struct = json_to_prost_struct(payload)
1359        .map_err(|err| anyhow!("failed to encode ClickHouse payload into Struct: {err}"))?;
1360    let offset =
1361        i64::try_from(raw_offset).map_err(|_| anyhow!("query offset exceeds i64 range"))?;
1362
1363    Ok(RunEventEnvelope {
1364        trace: Some(trace),
1365        recorded_at: Some(to_timestamp(recorded_at)),
1366        offset,
1367        severity: severity as i32,
1368        tags,
1369        raw: Some(raw_struct),
1370        body: None,
1371    })
1372}
1373
1374fn handle_ack(inflight: &mut VecDeque<i64>, last_ack: &mut i64, ack_offset: i64) {
1375    if ack_offset <= *last_ack {
1376        return;
1377    }
1378    *last_ack = ack_offset;
1379    while let Some(front) = inflight.front() {
1380        if *front <= ack_offset {
1381            inflight.pop_front();
1382        } else {
1383            break;
1384        }
1385    }
1386}
1387
1388fn passes_filters(
1389    config: &SubscriptionConfig,
1390    event: &models::OutboxEvent,
1391    severity: EventSeverity,
1392    tags: &[String],
1393) -> bool {
1394    if let Some(ref kinds) = config.kinds {
1395        if !kinds.contains(&event.kind.to_ascii_lowercase()) {
1396            return false;
1397        }
1398    }
1399
1400    if let Some(ref severities) = config.severities {
1401        if !severities.contains(&severity) {
1402            return false;
1403        }
1404    }
1405
1406    if let Some(ref step_filter) = config.step_ids {
1407        match event.step_id {
1408            Some(step_id) if step_filter.contains(&step_id) => {}
1409            _ => return false,
1410        }
1411    }
1412
1413    if let Some(ref tag_filter) = config.tags {
1414        if !tags
1415            .iter()
1416            .any(|tag| tag_filter.contains(&tag.to_ascii_lowercase()))
1417        {
1418            return false;
1419        }
1420    }
1421
1422    true
1423}
1424
1425fn derive_severity(kind: &str, payload: &Value) -> EventSeverity {
1426    let kind_lower = kind.to_ascii_lowercase();
1427    if kind_lower.contains("failed") || kind_lower.contains("error") {
1428        EventSeverity::Error
1429    } else if kind_lower.contains("denied") || kind_lower.contains("skipped") {
1430        EventSeverity::Warn
1431    } else if kind_lower.contains("debug") {
1432        EventSeverity::Debug
1433    } else if payload
1434        .get("severity")
1435        .and_then(Value::as_str)
1436        .map(|value| value.eq_ignore_ascii_case("warn"))
1437        .unwrap_or(false)
1438    {
1439        EventSeverity::Warn
1440    } else {
1441        EventSeverity::Info
1442    }
1443}
1444
1445fn severity_label(severity: EventSeverity) -> &'static str {
1446    match severity {
1447        EventSeverity::Trace => "trace",
1448        EventSeverity::Debug => "debug",
1449        EventSeverity::Info => "info",
1450        EventSeverity::Warn => "warn",
1451        EventSeverity::Error => "error",
1452        EventSeverity::Fatal => "fatal",
1453        EventSeverity::Unspecified => "unspecified",
1454    }
1455}
1456
1457fn derive_tags(
1458    kind: &str,
1459    step_id: Option<Uuid>,
1460    payload: &Value,
1461    severity: EventSeverity,
1462) -> Vec<String> {
1463    let mut tags = Vec::with_capacity(6);
1464    tags.push("tap".to_string());
1465    tags.push(format!("kind:{}", kind.to_ascii_lowercase()));
1466    tags.push(format!("severity:{}", severity_label(severity)));
1467
1468    if let Some(step_id) = step_id {
1469        tags.push("step".to_string());
1470        tags.push(format!("step_id:{step_id}"));
1471    } else {
1472        tags.push("run".to_string());
1473    }
1474
1475    if let Some(status) = payload.get("status").and_then(Value::as_str) {
1476        tags.push(format!("status:{}", status.to_ascii_lowercase()));
1477    }
1478    if let Some(effect) = payload.get("effect").and_then(Value::as_str) {
1479        tags.push(format!("effect:{}", effect.to_ascii_lowercase()));
1480    }
1481    tags
1482}
1483
1484fn build_response(
1485    config: &SubscriptionConfig,
1486    event: &models::OutboxEvent,
1487    severity: EventSeverity,
1488    tags: &[String],
1489    watermark: i64,
1490) -> Result<SubscribeRunResponse, Status> {
1491    let raw_struct = json_to_prost_struct(&event.payload).map_err(|err| {
1492        internal_error(format!(
1493            "failed to encode outbox payload {}: {err}",
1494            event.id
1495        ))
1496    })?;
1497
1498    let trace = build_trace_context(config.run_id, event.step_id, &event.payload);
1499
1500    let envelope = RunEventEnvelope {
1501        trace: Some(trace),
1502        recorded_at: Some(to_timestamp(event.created_at)),
1503        offset: event.id,
1504        severity: severity as i32,
1505        tags: tags.to_vec(),
1506        raw: Some(raw_struct),
1507        body: None,
1508    };
1509
1510    Ok(SubscribeRunResponse {
1511        offset: event.id,
1512        watermark,
1513        heartbeat: false,
1514        event: Some(envelope),
1515    })
1516}
1517
1518fn build_trace_context(run_id: Uuid, step_id: Option<Uuid>, payload: &Value) -> RunTraceContext {
1519    let mut ctx = RunTraceContext {
1520        trace_id: String::new(),
1521        span_id: String::new(),
1522        parent_span_id: String::new(),
1523        run: Some(tap::RunScope {
1524            run_id: run_id.to_string(),
1525            ..Default::default()
1526        }),
1527        step: None,
1528        tool_call: None,
1529        policy_decision: None,
1530        budget: None,
1531        seed: None,
1532        baggage: Vec::new(),
1533    };
1534
1535    if let Some(trace_value) = payload.get("trace").and_then(Value::as_object) {
1536        if let Some(trace_id) = trace_value.get("trace_id").and_then(Value::as_str) {
1537            ctx.trace_id = trace_id.to_string();
1538        }
1539        if let Some(span_id) = trace_value.get("span_id").and_then(Value::as_str) {
1540            ctx.span_id = span_id.to_string();
1541        }
1542        if let Some(parent_span_id) = trace_value.get("parent_span_id").and_then(Value::as_str) {
1543            ctx.parent_span_id = parent_span_id.to_string();
1544        }
1545        if let Some(seed) = trace_value.get("seed").and_then(Value::as_i64) {
1546            ctx.seed = Some(seed);
1547        }
1548        if let Some(run_obj) = trace_value.get("run").and_then(Value::as_object) {
1549            let run_id_value = run_obj
1550                .get("run_id")
1551                .and_then(Value::as_str)
1552                .map(str::to_owned)
1553                .unwrap_or_else(|| run_id.to_string());
1554            let mut run_scope = tap::RunScope {
1555                run_id: run_id_value,
1556                workspace_id: run_obj
1557                    .get("workspace_id")
1558                    .and_then(Value::as_str)
1559                    .unwrap_or_default()
1560                    .to_string(),
1561                app_id: run_obj
1562                    .get("app_id")
1563                    .and_then(Value::as_str)
1564                    .unwrap_or_default()
1565                    .to_string(),
1566                attempt_id: run_obj
1567                    .get("attempt_id")
1568                    .and_then(Value::as_str)
1569                    .unwrap_or_default()
1570                    .to_string(),
1571                labels: Default::default(),
1572            };
1573            if let Some(labels) = run_obj.get("labels").and_then(Value::as_object) {
1574                for (key, value) in labels {
1575                    if let Some(string) = value.as_str() {
1576                        run_scope.labels.insert(key.clone(), string.to_string());
1577                    }
1578                }
1579            }
1580            ctx.run = Some(run_scope);
1581        }
1582        if let Some(step_obj) = trace_value.get("step").and_then(Value::as_object) {
1583            let mut scope = tap::StepScope {
1584                step_id: step_obj
1585                    .get("step_id")
1586                    .and_then(Value::as_str)
1587                    .map(str::to_owned)
1588                    .unwrap_or_else(|| step_id.map(|id| id.to_string()).unwrap_or_default()),
1589                index: step_obj
1590                    .get("index")
1591                    .and_then(Value::as_u64)
1592                    .map(|value| value as u32)
1593                    .unwrap_or_default(),
1594                attempt: step_obj
1595                    .get("attempt")
1596                    .and_then(Value::as_u64)
1597                    .map(|value| value as u32)
1598                    .unwrap_or_default(),
1599                kind: step_obj
1600                    .get("kind")
1601                    .and_then(Value::as_str)
1602                    .unwrap_or_default()
1603                    .to_string(),
1604                scheduler: step_obj
1605                    .get("scheduler")
1606                    .and_then(Value::as_str)
1607                    .unwrap_or_default()
1608                    .to_string(),
1609            };
1610            if scope.step_id.is_empty() {
1611                if let Some(id) = step_id {
1612                    scope.step_id = id.to_string();
1613                }
1614            }
1615            ctx.step = Some(scope);
1616        } else if let Some(id) = step_id {
1617            ctx.step = Some(tap::StepScope {
1618                step_id: id.to_string(),
1619                ..Default::default()
1620            });
1621        }
1622        if let Some(tool_obj) = trace_value.get("tool_call").and_then(Value::as_object) {
1623            ctx.tool_call = Some(tap::ToolCallScope {
1624                tool_call_id: tool_obj
1625                    .get("tool_call_id")
1626                    .and_then(Value::as_str)
1627                    .unwrap_or_default()
1628                    .to_string(),
1629                tool_name: tool_obj
1630                    .get("tool_name")
1631                    .and_then(Value::as_str)
1632                    .unwrap_or_default()
1633                    .to_string(),
1634                tool_variant: tool_obj
1635                    .get("tool_variant")
1636                    .and_then(Value::as_str)
1637                    .unwrap_or_default()
1638                    .to_string(),
1639                provider: tool_obj
1640                    .get("provider")
1641                    .and_then(Value::as_str)
1642                    .unwrap_or_default()
1643                    .to_string(),
1644            });
1645        }
1646        if let Some(policy_obj) = trace_value
1647            .get("policy_decision")
1648            .and_then(Value::as_object)
1649        {
1650            ctx.policy_decision = Some(tap::PolicyDecisionScope {
1651                policy_decision_id: policy_obj
1652                    .get("policy_decision_id")
1653                    .and_then(Value::as_str)
1654                    .unwrap_or_default()
1655                    .to_string(),
1656                policy_pack_id: policy_obj
1657                    .get("policy_pack_id")
1658                    .and_then(Value::as_str)
1659                    .unwrap_or_default()
1660                    .to_string(),
1661                policy_name: policy_obj
1662                    .get("policy_name")
1663                    .and_then(Value::as_str)
1664                    .unwrap_or_default()
1665                    .to_string(),
1666                policy_version: policy_obj
1667                    .get("policy_version")
1668                    .and_then(Value::as_str)
1669                    .unwrap_or_default()
1670                    .to_string(),
1671            });
1672        }
1673        if let Some(budget_obj) = trace_value.get("budget").and_then(Value::as_object) {
1674            let mut budget_scope = tap::BudgetScope {
1675                budget_id: budget_obj
1676                    .get("budget_id")
1677                    .and_then(Value::as_str)
1678                    .unwrap_or_default()
1679                    .to_string(),
1680                budget_kind: budget_obj
1681                    .get("budget_kind")
1682                    .and_then(Value::as_str)
1683                    .unwrap_or_default()
1684                    .to_string(),
1685                amount_remaining: None,
1686                amount_allocated: None,
1687            };
1688            if let Some(value) = budget_obj.get("amount_remaining").and_then(Value::as_f64) {
1689                budget_scope.amount_remaining = Some(value);
1690            }
1691            if let Some(value) = budget_obj.get("amount_allocated").and_then(Value::as_f64) {
1692                budget_scope.amount_allocated = Some(value);
1693            }
1694            ctx.budget = Some(budget_scope);
1695        }
1696        if let Some(baggage_value) = trace_value.get("baggage") {
1697            ctx.baggage = parse_baggage(baggage_value);
1698        }
1699    } else if ctx.step.is_none() {
1700        if let Some(step_id) = step_id {
1701            ctx.step = Some(tap::StepScope {
1702                step_id: step_id.to_string(),
1703                ..Default::default()
1704            });
1705        }
1706    }
1707
1708    if ctx.run.is_none() {
1709        ctx.run = Some(tap::RunScope {
1710            run_id: run_id.to_string(),
1711            ..Default::default()
1712        });
1713    }
1714
1715    ctx
1716}
1717
1718fn parse_baggage(value: &Value) -> Vec<tap::BaggageItem> {
1719    if let Some(array) = value.as_array() {
1720        let mut items = Vec::with_capacity(array.len());
1721        for entry in array {
1722            if let Some(obj) = entry.as_object() {
1723                if let (Some(key), Some(value)) = (
1724                    obj.get("key").and_then(Value::as_str),
1725                    obj.get("value").and_then(Value::as_str),
1726                ) {
1727                    items.push(tap::BaggageItem {
1728                        key: key.to_string(),
1729                        value: value.to_string(),
1730                    });
1731                }
1732            }
1733        }
1734        return items;
1735    }
1736
1737    if let Some(header) = value.as_str() {
1738        return header
1739            .split(',')
1740            .filter_map(|entry| {
1741                let mut parts = entry.splitn(2, '=');
1742                let key = parts.next()?.trim();
1743                let remainder = parts.next()?.trim();
1744                let value = remainder
1745                    .split_once(';')
1746                    .map(|(val, _)| val.trim())
1747                    .unwrap_or(remainder);
1748                if key.is_empty() || value.is_empty() {
1749                    None
1750                } else {
1751                    Some(tap::BaggageItem {
1752                        key: key.to_string(),
1753                        value: value.to_string(),
1754                    })
1755                }
1756            })
1757            .collect();
1758    }
1759
1760    Vec::new()
1761}
1762
1763#[derive(Debug)]
1764struct SubmitRunSpec {
1765    dag_json: Value,
1766    inputs: Value,
1767    labels: HashMap<String, String>,
1768    seed: i64,
1769    traceparent: Option<String>,
1770    tracestate: Option<String>,
1771    baggage: Option<String>,
1772    debug: Option<Value>,
1773    breakpoints: Vec<String>,
1774}
1775
1776#[derive(Debug)]
1777enum SubmitRunOutcome {
1778    Created(Uuid),
1779    Reused(Uuid),
1780}
1781
1782#[derive(Debug)]
1783enum SubmitRunError {
1784    Invalid(String),
1785    Internal(String),
1786}
1787
1788impl std::fmt::Display for SubmitRunError {
1789    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1790        match self {
1791            SubmitRunError::Invalid(msg) => write!(f, "invalid run submission: {msg}"),
1792            SubmitRunError::Internal(msg) => write!(f, "internal run submission error: {msg}"),
1793        }
1794    }
1795}
1796
1797impl SubmitRunError {
1798    fn invalid(message: impl Into<String>) -> Self {
1799        Self::Invalid(message.into())
1800    }
1801
1802    fn internal(message: impl Into<String>) -> Self {
1803        Self::Internal(message.into())
1804    }
1805}
1806
1807fn format_schema_validation_error(err: SchemaValidationError) -> String {
1808    if err.details.is_empty() {
1809        err.schema.to_string()
1810    } else {
1811        err.details
1812            .iter()
1813            .map(|detail| detail.to_string())
1814            .collect::<Vec<_>>()
1815            .join("; ")
1816    }
1817}
1818
1819fn normalize_idempotency_key(raw: Option<&str>) -> Option<String> {
1820    raw.and_then(|value| {
1821        let trimmed = value.trim();
1822        if trimmed.is_empty() {
1823            None
1824        } else {
1825            Some(trimmed.to_owned())
1826        }
1827    })
1828}
1829
1830fn resolve_breakpoint(
1831    spec: &str,
1832    slug_map: &HashMap<String, Uuid>,
1833    step_ids: &HashSet<Uuid>,
1834) -> Result<Uuid, SubmitRunError> {
1835    let trimmed = spec.trim();
1836    if trimmed.is_empty() {
1837        return Err(SubmitRunError::invalid("breakpoint spec cannot be empty"));
1838    }
1839
1840    let (kind, target) = match trimmed.split_once(':') {
1841        Some((kind, rest)) => (kind.trim(), rest.trim()),
1842        None => ("step", trimmed),
1843    };
1844
1845    if target.is_empty() {
1846        return Err(SubmitRunError::invalid(format!(
1847            "breakpoint '{}' missing target",
1848            spec
1849        )));
1850    }
1851
1852    match kind.to_ascii_lowercase().as_str() {
1853        "step" => {
1854            if let Ok(uuid) = Uuid::parse_str(target) {
1855                if !step_ids.contains(&uuid) {
1856                    return Err(SubmitRunError::invalid(format!(
1857                        "breakpoint references unknown step id '{}'",
1858                        target
1859                    )));
1860                }
1861                return Ok(uuid);
1862            }
1863
1864            let key = target.to_ascii_lowercase();
1865            if let Some(uuid) = slug_map.get(&key) {
1866                return Ok(*uuid);
1867            }
1868
1869            Err(SubmitRunError::invalid(format!(
1870                "breakpoint references unknown step slug '{}'",
1871                target
1872            )))
1873        }
1874        other => Err(SubmitRunError::invalid(format!(
1875            "unsupported breakpoint kind '{}'",
1876            other
1877        ))),
1878    }
1879}
1880
1881async fn submit_run_impl(
1882    storage: &Storage,
1883    audit: &Audit,
1884    spec: SubmitRunSpec,
1885    idempotency_key: Option<String>,
1886    actor: &str,
1887) -> Result<SubmitRunOutcome, SubmitRunError> {
1888    if let Some(ref key) = idempotency_key {
1889        if let Some(existing_run) =
1890            storage
1891                .fetch_run_by_idempotency_key(key)
1892                .await
1893                .map_err(|err| {
1894                    SubmitRunError::internal(format!("failed to check idempotency key: {err}"))
1895                })?
1896        {
1897            info!(
1898                run_id = %existing_run.run_id,
1899                idempotency_key = %key,
1900                "reusing existing run for idempotent submission"
1901            );
1902            return Ok(SubmitRunOutcome::Reused(existing_run.run_id));
1903        }
1904    }
1905
1906    let SubmitRunSpec {
1907        dag_json,
1908        inputs,
1909        labels,
1910        seed,
1911        traceparent,
1912        tracestate,
1913        baggage,
1914        debug,
1915        breakpoints,
1916    } = spec;
1917
1918    if seed <= 0 {
1919        return Err(SubmitRunError::invalid(
1920            "seed must be a positive integer to guarantee deterministic replay",
1921        ));
1922    }
1923
1924    let steps_array = dag_json
1925        .get("steps")
1926        .and_then(Value::as_array)
1927        .ok_or_else(|| SubmitRunError::invalid("dag_json.steps must be an array"))?;
1928
1929    let inputs_value = match inputs {
1930        Value::Null => Value::Object(Map::new()),
1931        Value::Object(map) => Value::Object(map),
1932        _ => return Err(SubmitRunError::invalid("inputs must be a JSON object")),
1933    };
1934
1935    let labels_value = labels_map_to_value(&labels);
1936
1937    let run_spec_instance = json!({
1938        "dag_json": dag_json.clone(),
1939        "inputs": inputs_value.clone(),
1940        "labels": labels_value.clone(),
1941        "seed": seed,
1942        "breakpoints": breakpoints.clone(),
1943    });
1944
1945    if let Err(err) = validate_run_spec(&run_spec_instance) {
1946        return Err(SubmitRunError::invalid(format!(
1947            "run spec schema validation failed: {}",
1948            format_schema_validation_error(err)
1949        )));
1950    }
1951
1952    let mut debug_map = match debug {
1953        Some(Value::Object(map)) => map,
1954        Some(_) => {
1955            return Err(SubmitRunError::invalid(
1956                "debug metadata must be a JSON object",
1957            ))
1958        }
1959        None => Map::new(),
1960    };
1961
1962    let mut ctx_map = Map::new();
1963    ctx_map.insert("inputs".to_string(), inputs_value.clone());
1964    ctx_map.insert("labels".to_string(), labels_value.clone());
1965    if traceparent.is_some() || tracestate.is_some() || baggage.is_some() {
1966        let mut trace_obj = Map::new();
1967        if let Some(tp) = traceparent {
1968            trace_obj.insert("traceparent".to_string(), Value::String(tp));
1969        }
1970        if let Some(ts) = tracestate {
1971            trace_obj.insert("tracestate".to_string(), Value::String(ts));
1972        }
1973        if let Some(bg) = baggage {
1974            trace_obj.insert("baggage".to_string(), Value::String(bg));
1975        }
1976        ctx_map.insert("trace".to_string(), Value::Object(trace_obj));
1977    }
1978
1979    let submission_time = Utc::now();
1980
1981    struct PendingStep {
1982        idx: i32,
1983        step_id: Uuid,
1984        priority: i16,
1985        spec: Value,
1986        max_attempts: i32,
1987        retry_backoff_ms: i64,
1988        retry_backoff_factor: f64,
1989        deadline_at: Option<DateTime<Utc>>,
1990        estimated_tokens: i64,
1991        estimated_cost: f64,
1992        compensation_target: Option<String>,
1993    }
1994
1995    let mut pending_steps = Vec::with_capacity(steps_array.len());
1996    let mut seen_step_ids = HashSet::with_capacity(steps_array.len());
1997    let mut slug_to_step: HashMap<String, Uuid> = HashMap::new();
1998    let mut slug_entries: Vec<(String, Uuid)> = Vec::new();
1999
2000    for (idx, step_value) in steps_array.iter().enumerate() {
2001        if let Err(err) = validate_step_spec(step_value) {
2002            let detail = err
2003                .details
2004                .iter()
2005                .map(|err| err.to_string())
2006                .collect::<Vec<_>>()
2007                .join("; ");
2008            return Err(SubmitRunError::invalid(format!(
2009                "step schema validation failed: {detail}"
2010            )));
2011        }
2012
2013        let step_obj = step_value
2014            .as_object()
2015            .ok_or_else(|| SubmitRunError::invalid("each step must be a JSON object"))?;
2016
2017        let step_id_str = step_obj
2018            .get("id")
2019            .and_then(Value::as_str)
2020            .ok_or_else(|| SubmitRunError::invalid("step.id must be a UUID string"))?;
2021        let step_id = Uuid::parse_str(step_id_str)
2022            .map_err(|err| SubmitRunError::invalid(format!("invalid step.id: {err}")))?;
2023
2024        if !seen_step_ids.insert(step_id) {
2025            return Err(SubmitRunError::invalid(format!(
2026                "duplicate step.id '{}' detected in dag_json.steps",
2027                step_id
2028            )));
2029        }
2030
2031        if let Some(slug_value) = step_obj.get("slug").and_then(Value::as_str) {
2032            let trimmed = slug_value.trim();
2033            if trimmed.is_empty() {
2034                return Err(SubmitRunError::invalid("step.slug cannot be empty"));
2035            }
2036            let key = trimmed.to_ascii_lowercase();
2037            if slug_to_step.insert(key.clone(), step_id).is_some() {
2038                return Err(SubmitRunError::invalid(format!(
2039                    "duplicate step.slug '{}' detected in dag_json.steps",
2040                    trimmed
2041                )));
2042            }
2043            slug_entries.push((trimmed.to_string(), step_id));
2044        }
2045
2046        let priority = extract_priority(step_obj);
2047
2048        let execution_obj = step_obj.get("execution").and_then(Value::as_object);
2049        let max_attempts = execution_obj
2050            .and_then(|exec| exec.get("max_attempts").and_then(Value::as_i64))
2051            .and_then(|v| i32::try_from(v).ok())
2052            .filter(|v| *v >= 1)
2053            .unwrap_or(1);
2054        let retry_backoff_ms = execution_obj
2055            .and_then(|exec| exec.get("retry_backoff_ms").and_then(Value::as_i64))
2056            .map(|v| v.max(0))
2057            .unwrap_or(0);
2058        let retry_backoff_factor = execution_obj
2059            .and_then(|exec| exec.get("retry_backoff_factor").and_then(Value::as_f64))
2060            .filter(|v| *v >= 1.0)
2061            .unwrap_or(1.0);
2062        let deadline_at = execution_obj
2063            .and_then(|exec| exec.get("deadline_ms").and_then(Value::as_i64))
2064            .filter(|ms| *ms > 0)
2065            .map(|ms| submission_time + chrono::Duration::milliseconds(ms));
2066
2067        let cost_hint =
2068            execution_obj.and_then(|exec| exec.get("cost_hint").and_then(Value::as_object));
2069        let estimated_tokens = cost_hint
2070            .and_then(|hint| hint.get("tokens").and_then(Value::as_i64))
2071            .map(|v| v.max(0))
2072            .unwrap_or(0);
2073        let estimated_cost = cost_hint
2074            .and_then(|hint| hint.get("usd").and_then(Value::as_f64))
2075            .map(|v| v.max(0.0))
2076            .unwrap_or(0.0);
2077
2078        let compensation_target = execution_obj
2079            .and_then(|exec| exec.get("compensation").and_then(Value::as_object))
2080            .and_then(|comp| comp.get("step").and_then(Value::as_str))
2081            .map(|s| s.to_string());
2082
2083        pending_steps.push(PendingStep {
2084            idx: idx as i32,
2085            step_id,
2086            priority,
2087            spec: step_value.clone(),
2088            max_attempts,
2089            retry_backoff_ms,
2090            retry_backoff_factor,
2091            deadline_at,
2092            estimated_tokens,
2093            estimated_cost,
2094            compensation_target,
2095        });
2096    }
2097    let mut incoming_counts: HashMap<Uuid, i32> =
2098        pending_steps.iter().map(|step| (step.step_id, 0)).collect();
2099
2100    let mut edges: Vec<(Uuid, Uuid)> = Vec::new();
2101    if let Some(edges_array) = dag_json.get("edges").and_then(Value::as_array) {
2102        let mut seen_edges = HashSet::with_capacity(edges_array.len());
2103        for edge_value in edges_array {
2104            let pair = edge_value.as_array().ok_or_else(|| {
2105                SubmitRunError::invalid("dag_json.edges entries must be [from, to] arrays")
2106            })?;
2107            if pair.len() != 2 {
2108                return Err(SubmitRunError::invalid(
2109                    "dag_json.edges entries must contain exactly two elements",
2110                ));
2111            }
2112
2113            let from_str = pair[0]
2114                .as_str()
2115                .ok_or_else(|| SubmitRunError::invalid("edge source must be a UUID string"))?;
2116            let to_str = pair[1]
2117                .as_str()
2118                .ok_or_else(|| SubmitRunError::invalid("edge target must be a UUID string"))?;
2119
2120            let from = Uuid::parse_str(from_str).map_err(|err| {
2121                SubmitRunError::invalid(format!("invalid edge source UUID: {err}"))
2122            })?;
2123            let to = Uuid::parse_str(to_str).map_err(|err| {
2124                SubmitRunError::invalid(format!("invalid edge target UUID: {err}"))
2125            })?;
2126
2127            if from == to {
2128                return Err(SubmitRunError::invalid(
2129                    "edge cannot reference the same step twice",
2130                ));
2131            }
2132            if !seen_step_ids.contains(&from) || !seen_step_ids.contains(&to) {
2133                return Err(SubmitRunError::invalid("edge references unknown step id"));
2134            }
2135            if !seen_edges.insert((from, to)) {
2136                return Err(SubmitRunError::invalid(
2137                    "duplicate edge detected in dag_json.edges",
2138                ));
2139            }
2140
2141            let entry = incoming_counts.get_mut(&to).ok_or_else(|| {
2142                SubmitRunError::invalid("edge target not present in dag_json.steps")
2143            })?;
2144            let next_count = (*entry).checked_add(1).ok_or_else(|| {
2145                SubmitRunError::invalid("too many incoming edges for a single step")
2146            })?;
2147            *entry = next_count;
2148
2149            edges.push((from, to));
2150        }
2151    }
2152
2153    let mut breakpoint_payloads: Vec<Value> = Vec::new();
2154    let mut breakpoint_specs: Vec<String> = Vec::new();
2155    let mut breakpoint_ids: HashSet<Uuid> = HashSet::new();
2156    for bp in &breakpoints {
2157        let resolved = resolve_breakpoint(bp, &slug_to_step, &seen_step_ids)?;
2158        if !breakpoint_ids.insert(resolved) {
2159            return Err(SubmitRunError::invalid(format!(
2160                "duplicate breakpoint targeting step {}",
2161                resolved
2162            )));
2163        }
2164
2165        let mut record = Map::new();
2166        record.insert("spec".to_string(), Value::String(bp.clone()));
2167        record.insert("step_id".to_string(), Value::String(resolved.to_string()));
2168        breakpoint_payloads.push(Value::Object(record));
2169        breakpoint_specs.push(bp.clone());
2170    }
2171
2172    if !breakpoint_payloads.is_empty() {
2173        debug_map.insert(
2174            "breakpoints".to_string(),
2175            Value::Array(breakpoint_payloads.clone()),
2176        );
2177        let specs = breakpoint_specs
2178            .iter()
2179            .cloned()
2180            .map(Value::String)
2181            .collect::<Vec<_>>();
2182        debug_map.insert("breakpoint_specs".to_string(), Value::Array(specs));
2183    }
2184
2185    if !slug_entries.is_empty() {
2186        let mut slugs = Map::new();
2187        for (slug, step_id) in &slug_entries {
2188            slugs.insert(slug.clone(), Value::String(step_id.to_string()));
2189        }
2190        debug_map.insert("slugs".to_string(), Value::Object(slugs));
2191    }
2192
2193    debug_map
2194        .entry("active_pause".to_string())
2195        .or_insert(Value::Null);
2196
2197    if !debug_map.is_empty() {
2198        ctx_map.insert("debug".to_string(), Value::Object(debug_map.clone()));
2199    }
2200
2201    let input_ctx = Value::Object(ctx_map);
2202
2203    let run_id = Uuid::new_v4();
2204    let mut steps = Vec::with_capacity(pending_steps.len());
2205    for pending in pending_steps {
2206        let PendingStep {
2207            idx,
2208            step_id,
2209            priority,
2210            spec,
2211            max_attempts,
2212            retry_backoff_ms,
2213            retry_backoff_factor,
2214            deadline_at,
2215            estimated_tokens,
2216            estimated_cost,
2217            compensation_target,
2218        } = pending;
2219
2220        let dep_count = *incoming_counts.get(&step_id).unwrap_or(&0);
2221        let status = if dep_count == 0 {
2222            models::StepStatus::Queued
2223        } else {
2224            models::StepStatus::Blocked
2225        };
2226
2227        let compensation_step = if let Some(target) = compensation_target {
2228            let trimmed = target.trim();
2229            if trimmed.is_empty() {
2230                return Err(SubmitRunError::invalid(format!(
2231                    "compensation step reference cannot be empty for step {}",
2232                    step_id
2233                )));
2234            }
2235
2236            let resolved = if let Ok(uuid) = Uuid::parse_str(trimmed) {
2237                if !seen_step_ids.contains(&uuid) {
2238                    return Err(SubmitRunError::invalid(format!(
2239                        "compensation step '{}' not found in dag_json.steps",
2240                        trimmed
2241                    )));
2242                }
2243                uuid
2244            } else {
2245                let key = trimmed.to_ascii_lowercase();
2246                *slug_to_step.get(&key).ok_or_else(|| {
2247                    SubmitRunError::invalid(format!(
2248                        "compensation step '{}' not found in dag_json.steps",
2249                        trimmed
2250                    ))
2251                })?
2252            };
2253
2254            if resolved == step_id {
2255                return Err(SubmitRunError::invalid(format!(
2256                    "step {} cannot reference itself as a compensation step",
2257                    step_id
2258                )));
2259            }
2260
2261            Some(resolved)
2262        } else {
2263            None
2264        };
2265
2266        steps.push(models::NewStep {
2267            run_id,
2268            step_id,
2269            idx,
2270            priority,
2271            spec_json: spec,
2272            status,
2273            attempt: 0,
2274            input_snapshot: None,
2275            provider: None,
2276            provider_version: None,
2277            max_attempts,
2278            retry_backoff_ms,
2279            retry_backoff_factor,
2280            not_before: None,
2281            deadline_at,
2282            pending_dependencies: dep_count,
2283            total_dependencies: dep_count,
2284            estimated_tokens,
2285            estimated_cost,
2286            actual_tokens: None,
2287            actual_cost: None,
2288            checkpoint: None,
2289            compensation_step,
2290            compensation_scheduled: false,
2291        });
2292    }
2293
2294    let new_run = models::NewRun {
2295        run_id,
2296        status: models::RunStatus::Pending,
2297        dag_json: dag_json.clone(),
2298        input_ctx,
2299        seed,
2300        idempotency_key: idempotency_key.clone(),
2301    };
2302
2303    let stored_run_id = storage
2304        .insert_run_with_steps(new_run, &steps, &edges)
2305        .await
2306        .map_err(|err| {
2307            error!(error = ?err, "failed to persist run");
2308            SubmitRunError::internal(format!("failed to persist run: {err}"))
2309        })?;
2310
2311    info!(run_id = %stored_run_id, "accepted run submission");
2312
2313    let tenant_id = labels
2314        .get("tenant_id")
2315        .or_else(|| labels.get("tenant"))
2316        .and_then(|value| Uuid::parse_str(value).ok());
2317    let audit_resource = json!({
2318        "run_id": stored_run_id.to_string(),
2319        "idempotency_key": idempotency_key,
2320        "labels": labels,
2321    });
2322
2323    if let Err(err) = audit
2324        .append(actor, tenant_id, "run.submit", &audit_resource)
2325        .await
2326    {
2327        warn!(
2328            error = %err,
2329            run_id = %stored_run_id,
2330            "failed to append audit log for run submission"
2331        );
2332    }
2333
2334    Ok(SubmitRunOutcome::Created(stored_run_id))
2335}
2336
2337#[tonic::async_trait]
2338impl RuntimeService for RuntimeApiService {
2339    async fn submit_run(
2340        &self,
2341        request: Request<runtime::SubmitRunRequest>,
2342    ) -> Result<Response<runtime::SubmitRunResponse>, Status> {
2343        let metadata = request.metadata().clone();
2344        let role = self.require_role(&metadata, Role::Writer).await?;
2345        let req = request.into_inner();
2346        let runtime::SubmitRunRequest {
2347            spec,
2348            idempotency_key: raw_idempotency_key,
2349        } = req;
2350
2351        let idempotency_key = normalize_idempotency_key(Some(raw_idempotency_key.as_str()));
2352
2353        let spec = spec.ok_or_else(|| invalid_argument("spec is required"))?;
2354
2355        let dag_json = parse_dag_json(&spec.dag_json)?;
2356        let inputs_value = spec
2357            .inputs
2358            .as_ref()
2359            .map(prost_struct_to_json)
2360            .unwrap_or_else(|| Value::Object(Map::new()));
2361
2362        let traceparent = metadata
2363            .get("traceparent")
2364            .and_then(|value| value.to_str().ok())
2365            .map(|s| s.to_string());
2366        let tracestate = metadata
2367            .get("tracestate")
2368            .and_then(|value| value.to_str().ok())
2369            .map(|s| s.to_string());
2370        let baggage = metadata
2371            .get("baggage")
2372            .and_then(|value| value.to_str().ok())
2373            .map(|s| s.to_string());
2374
2375        let submission = SubmitRunSpec {
2376            dag_json,
2377            inputs: inputs_value,
2378            labels: spec.labels.clone(),
2379            seed: spec.seed,
2380            traceparent,
2381            tracestate,
2382            baggage,
2383            debug: None,
2384            breakpoints: spec.breakpoints.clone(),
2385        };
2386
2387        let actor = Self::actor_from_metadata(&metadata, role);
2388
2389        let outcome = submit_run_impl(
2390            self.storage.as_ref(),
2391            self.audit.as_ref(),
2392            submission,
2393            idempotency_key,
2394            &actor,
2395        )
2396        .await
2397        .map_err(|err| match err {
2398            SubmitRunError::Invalid(message) => invalid_argument(message),
2399            SubmitRunError::Internal(message) => internal_error(message),
2400        })?;
2401
2402        let run_id = match outcome {
2403            SubmitRunOutcome::Created(run_id) | SubmitRunOutcome::Reused(run_id) => run_id,
2404        };
2405
2406        Ok(Response::new(runtime::SubmitRunResponse {
2407            run_id: run_id.to_string(),
2408        }))
2409    }
2410
2411    async fn get_run(
2412        &self,
2413        request: Request<runtime::GetRunRequest>,
2414    ) -> Result<Response<runtime::GetRunResponse>, Status> {
2415        let metadata = request.metadata().clone();
2416        self.require_role(&metadata, Role::Reader).await?;
2417        let req = request.into_inner();
2418        let run_id = Uuid::parse_str(&req.run_id)
2419            .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2420
2421        let run = self
2422            .storage
2423            .fetch_run(run_id)
2424            .await
2425            .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
2426            .ok_or_else(|| Status::not_found("run not found"))?;
2427
2428        let steps = self
2429            .storage
2430            .fetch_steps_for_run(run_id)
2431            .await
2432            .map_err(|err| internal_error(format!("failed to fetch steps: {err}")))?;
2433
2434        let run_detail = build_run_detail(run, steps)?;
2435
2436        Ok(Response::new(runtime::GetRunResponse {
2437            run: Some(run_detail),
2438        }))
2439    }
2440
2441    type TapRunStream = Pin<Box<dyn Stream<Item = Result<runtime::RunEvent, Status>> + Send>>;
2442
2443    async fn tap_run(
2444        &self,
2445        request: Request<runtime::TapRunRequest>,
2446    ) -> Result<Response<Self::TapRunStream>, Status> {
2447        let metadata = request.metadata().clone();
2448        self.require_role(&metadata, Role::Reader).await?;
2449        let req = request.into_inner();
2450        let run_uuid = Uuid::parse_str(&req.run_id)
2451            .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2452
2453        // Ensure the run exists before entering the stream loop.
2454        if self
2455            .storage
2456            .fetch_run(run_uuid)
2457            .await
2458            .map_err(|err| internal_error(format!("failed to verify run: {err}")))?
2459            .is_none()
2460        {
2461            return Err(Status::not_found("run not found"));
2462        }
2463
2464        let storage = Arc::clone(&self.storage);
2465        let (tx, rx) = mpsc::channel::<Result<runtime::RunEvent, Status>>(32);
2466
2467        tokio::spawn(async move {
2468            let mut last_id: i64 = 0;
2469
2470            loop {
2471                if tx.is_closed() {
2472                    break;
2473                }
2474
2475                match storage
2476                    .fetch_outbox_for_run_since(run_uuid, last_id, 256)
2477                    .await
2478                {
2479                    Ok(events) => {
2480                        if events.is_empty() {
2481                            sleep(Duration::from_millis(500)).await;
2482                            continue;
2483                        }
2484
2485                        for event in events {
2486                            last_id = event.id;
2487                            match run_event_from_outbox(event) {
2488                                Ok(run_event) => {
2489                                    if tx.send(Ok(run_event)).await.is_err() {
2490                                        return;
2491                                    }
2492                                }
2493                                Err(status) => {
2494                                    let _ = tx.send(Err(status)).await;
2495                                    return;
2496                                }
2497                            }
2498                        }
2499                    }
2500                    Err(err) => {
2501                        warn!(run_id = %run_uuid, error = %err, "tap stream failed to read outbox events");
2502                        let status = internal_error(format!("failed to fetch run events: {err}"));
2503                        let _ = tx.send(Err(status)).await;
2504                        break;
2505                    }
2506                }
2507            }
2508        });
2509
2510        let stream = ReceiverStream::new(rx);
2511        Ok(Response::new(Box::pin(stream) as Self::TapRunStream))
2512    }
2513
2514    async fn replay_run(
2515        &self,
2516        request: Request<runtime::ReplayRunRequest>,
2517    ) -> Result<Response<runtime::ReplayRunResponse>, Status> {
2518        let metadata = request.metadata().clone();
2519        self.require_role(&metadata, Role::Writer).await?;
2520        let req = request.into_inner();
2521        let run_uuid = Uuid::parse_str(&req.run_id)
2522            .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2523
2524        if self
2525            .storage
2526            .fetch_run(run_uuid)
2527            .await
2528            .map_err(|err| internal_error(format!("failed to load run: {err}")))?
2529            .is_none()
2530        {
2531            return Err(Status::not_found("run not found"));
2532        }
2533
2534        let strategy = match runtime::ReplayMode::try_from(req.mode).ok() {
2535            Some(runtime::ReplayMode::Unspecified)
2536            | Some(runtime::ReplayMode::Full)
2537            | Some(runtime::ReplayMode::Diff) => runtime::ReplayStrategy::Mocked,
2538            Some(runtime::ReplayMode::StepOnly) => {
2539                return Err(Status::unimplemented(
2540                    "Replay mode STEP_ONLY is not yet supported",
2541                ));
2542            }
2543            None => {
2544                return Err(invalid_argument("unrecognized replay mode"));
2545            }
2546        };
2547
2548        let steps = self
2549            .storage
2550            .fetch_steps_for_run(run_uuid)
2551            .await
2552            .map_err(|err| internal_error(format!("failed to load steps for replay: {err}")))?;
2553        let attempts = self
2554            .storage
2555            .fetch_step_attempts_for_run(run_uuid)
2556            .await
2557            .map_err(|err| {
2558                internal_error(format!("failed to load step attempts for replay: {err}"))
2559            })?;
2560
2561        let mut attempts_by_step: HashMap<Uuid, Vec<models::StepAttempt>> = HashMap::new();
2562        for attempt in attempts {
2563            attempts_by_step
2564                .entry(attempt.step_id)
2565                .or_default()
2566                .push(attempt);
2567        }
2568        for entries in attempts_by_step.values_mut() {
2569            entries.sort_by_key(|attempt| attempt.attempt);
2570        }
2571
2572        let replay_mode =
2573            runtime::ReplayMode::try_from(req.mode).unwrap_or(runtime::ReplayMode::Full);
2574        let diff_mode = matches!(replay_mode, runtime::ReplayMode::Diff);
2575
2576        let mut proto_steps = Vec::with_capacity(steps.len());
2577        let mut all_within = true;
2578
2579        for step in steps {
2580            let attempts_for_step = attempts_by_step.get(&step.step_id);
2581            let latest_attempt = attempts_for_step.and_then(|entries| entries.last());
2582
2583            let recorded_snapshot = latest_attempt
2584                .and_then(|attempt| attempt.output_snapshot.clone())
2585                .or_else(|| step.output_snapshot.clone())
2586                .ok_or_else(|| {
2587                    Status::failed_precondition(format!(
2588                        "step {} missing output_snapshot; cannot replay",
2589                        step.step_id
2590                    ))
2591                })?;
2592
2593            let baseline_attempt = if diff_mode {
2594                attempts_for_step.and_then(|entries| {
2595                    if entries.len() >= 2 {
2596                        entries.get(entries.len() - 2)
2597                    } else {
2598                        None
2599                    }
2600                })
2601            } else {
2602                latest_attempt
2603            };
2604
2605            let baseline_output = baseline_attempt
2606                .and_then(|attempt| attempt.output_snapshot.clone())
2607                .unwrap_or_else(|| recorded_snapshot.clone());
2608
2609            let current_output = if diff_mode {
2610                recorded_snapshot.clone()
2611            } else {
2612                step.output_json
2613                    .clone()
2614                    .unwrap_or_else(|| recorded_snapshot.clone())
2615            };
2616
2617            let drift_result = compute_drift(
2618                &baseline_output,
2619                &current_output,
2620                DEFAULT_TOKEN_TOLERANCE,
2621                DEFAULT_COST_TOLERANCE,
2622            );
2623            if !drift_result.within_tolerance {
2624                all_within = false;
2625            }
2626
2627            let snapshot_value = json_to_prost_value(&recorded_snapshot)
2628                .map_err(|err| internal_error(format!("failed to encode snapshot: {err}")))?;
2629
2630            let recorded_meta = wrap_attempt_output(baseline_attempt, &baseline_output);
2631            let current_meta = wrap_attempt_output(latest_attempt, &current_output);
2632
2633            let drift_proto = runtime::ReplayStepDrift {
2634                recorded: Some(json_to_prost_value(&recorded_meta).map_err(|err| {
2635                    internal_error(format!("failed to encode recorded drift snapshot: {err}"))
2636                })?),
2637                current: Some(json_to_prost_value(&current_meta).map_err(|err| {
2638                    internal_error(format!("failed to encode current drift snapshot: {err}"))
2639                })?),
2640                token_delta: drift_result.token_delta,
2641                cost_delta: drift_result.cost_delta,
2642                within_tolerance: drift_result.within_tolerance,
2643            };
2644
2645            proto_steps.push(runtime::ReplayStep {
2646                step_id: step.step_id.to_string(),
2647                snapshot: Some(snapshot_value),
2648                drift: Some(drift_proto),
2649            });
2650        }
2651
2652        let response = runtime::ReplayRunResponse {
2653            run_id: req.run_id,
2654            all_within_tolerance: all_within,
2655            strategy: strategy as i32,
2656            steps: proto_steps,
2657        };
2658
2659        Ok(Response::new(response))
2660    }
2661
2662    async fn update_breakpoints(
2663        &self,
2664        request: Request<runtime::UpdateBreakpointsRequest>,
2665    ) -> Result<Response<runtime::UpdateBreakpointsResponse>, Status> {
2666        let metadata = request.metadata().clone();
2667        let role = self.require_role(&metadata, Role::Writer).await?;
2668        let actor = Self::actor_from_metadata(&metadata, role);
2669        let req = request.into_inner();
2670
2671        let run_uuid = Uuid::parse_str(&req.run_id)
2672            .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2673
2674        if !req.clear_all && req.add.is_empty() && req.remove.is_empty() {
2675            return Err(invalid_argument(
2676                "update_breakpoints requires add/remove entries or clear_all",
2677            ));
2678        }
2679
2680        let run = self
2681            .storage
2682            .fetch_run(run_uuid)
2683            .await
2684            .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
2685            .ok_or_else(|| Status::not_found("run not found"))?;
2686
2687        let step_ids = collect_step_ids(&run);
2688
2689        let ctx_value = run.input_ctx.clone();
2690        let mut ctx_map = match ctx_value {
2691            Value::Object(map) => map,
2692            Value::Null => Map::new(),
2693            other => {
2694                return Err(internal_error(format!(
2695                    "run input context must be an object, found {other}"
2696                )))
2697            }
2698        };
2699
2700        let debug_entry = ctx_map
2701            .entry("debug".to_string())
2702            .or_insert_with(|| Value::Object(Map::new()));
2703        let debug_map = debug_entry.as_object_mut().ok_or_else(|| {
2704            internal_error("run debug context is not an object; unable to update breakpoints")
2705        })?;
2706
2707        let mut breakpoints = load_debug_breakpoints(debug_map);
2708        let existing_slug_entries = load_slug_map(debug_map);
2709
2710        let mut slug_lookup: HashMap<String, Uuid> = existing_slug_entries
2711            .iter()
2712            .map(|(slug, id)| (slug.to_ascii_lowercase(), *id))
2713            .collect();
2714        if slug_lookup.is_empty() {
2715            for (slug, id) in collect_slug_entries(&run) {
2716                slug_lookup.insert(slug.to_ascii_lowercase(), id);
2717            }
2718        }
2719
2720        let slug_entries_store: Vec<(String, Uuid)> = if existing_slug_entries.is_empty() {
2721            collect_slug_entries(&run)
2722        } else {
2723            existing_slug_entries
2724                .iter()
2725                .map(|(slug, id)| (slug.clone(), *id))
2726                .collect()
2727        };
2728
2729        if req.clear_all {
2730            breakpoints.clear();
2731        }
2732
2733        let removal_specs: HashSet<String> = req
2734            .remove
2735            .iter()
2736            .map(|s| s.trim().to_string())
2737            .filter(|s| !s.is_empty())
2738            .collect();
2739
2740        let mut removal_ids: HashSet<Uuid> = HashSet::new();
2741        for spec in &req.remove {
2742            if let Ok(step_id) = resolve_breakpoint(spec, &slug_lookup, &step_ids) {
2743                removal_ids.insert(step_id);
2744            }
2745        }
2746
2747        if !removal_specs.is_empty() || !removal_ids.is_empty() {
2748            breakpoints.retain(|(spec, step_id)| {
2749                let spec_trim = spec.trim();
2750                let matches_spec = removal_specs.contains(spec_trim);
2751                let matches_id = removal_ids.contains(step_id);
2752                !(matches_spec || matches_id)
2753            });
2754        }
2755
2756        for spec in &req.add {
2757            let step_id = resolve_breakpoint(spec, &slug_lookup, &step_ids)
2758                .map_err(|err| invalid_argument(err.to_string()))?;
2759            breakpoints.retain(|(_, existing)| existing != &step_id);
2760            breakpoints.push((spec.trim().to_string(), step_id));
2761        }
2762
2763        breakpoints.sort_by(|a, b| a.0.cmp(&b.0));
2764        store_debug_breakpoints(debug_map, &breakpoints);
2765        if !slug_entries_store.is_empty() {
2766            store_slug_map(debug_map, &slug_entries_store);
2767        }
2768
2769        let mut tx = self
2770            .storage
2771            .pool()
2772            .begin()
2773            .await
2774            .map_err(|err| internal_error(format!("failed to begin transaction: {err}")))?;
2775
2776        self.storage
2777            .update_run_input_ctx_tx(&mut tx, run_uuid, Value::Object(ctx_map.clone()))
2778            .await
2779            .map_err(|err| {
2780                internal_error(format!("failed to persist breakpoint changes: {err}"))
2781            })?;
2782
2783        let payload_breakpoints = breakpoints
2784            .iter()
2785            .map(|(spec, step_id)| {
2786                json!({
2787                    "spec": spec,
2788                    "step_id": step_id.to_string(),
2789                })
2790            })
2791            .collect::<Vec<_>>();
2792
2793        self.storage
2794            .enqueue_outbox_event(
2795                &mut tx,
2796                NewOutboxEvent {
2797                    run_id: run_uuid,
2798                    step_id: None,
2799                    kind: "run_breakpoints_updated".to_string(),
2800                    payload: json!({
2801                        "run_id": run_uuid.to_string(),
2802                        "breakpoints": payload_breakpoints,
2803                        "added": req.add,
2804                        "removed": req.remove,
2805                        "cleared": req.clear_all,
2806                    }),
2807                },
2808            )
2809            .await
2810            .map_err(|err| internal_error(format!("failed to enqueue outbox event: {err}")))?;
2811
2812        tx.commit()
2813            .await
2814            .map_err(|err| internal_error(format!("failed to commit breakpoint update: {err}")))?;
2815
2816        if let Err(err) = self
2817            .audit
2818            .append(
2819                &actor,
2820                self.storage
2821                    .fetch_run_tenant(run_uuid)
2822                    .await
2823                    .ok()
2824                    .flatten()
2825                    .and_then(|tenant| Uuid::parse_str(&tenant).ok()),
2826                "debug.breakpoints.update",
2827                &json!({
2828                    "run_id": run_uuid.to_string(),
2829                    "added": req.add,
2830                    "removed": req.remove,
2831                    "cleared": req.clear_all,
2832                }),
2833            )
2834            .await
2835        {
2836            warn!(error = %err, run = %run_uuid, "failed to append audit record for breakpoint update");
2837        }
2838
2839        Ok(Response::new(runtime::UpdateBreakpointsResponse {
2840            breakpoints: breakpoints.into_iter().map(|(spec, _)| spec).collect(),
2841        }))
2842    }
2843
2844    async fn resume_run(
2845        &self,
2846        request: Request<runtime::ResumeRunRequest>,
2847    ) -> Result<Response<runtime::ResumeRunResponse>, Status> {
2848        let metadata = request.metadata().clone();
2849        let role = self.require_role(&metadata, Role::Writer).await?;
2850        let actor = Self::actor_from_metadata(&metadata, role);
2851        let req = request.into_inner();
2852
2853        let run_uuid = Uuid::parse_str(&req.run_id)
2854            .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2855
2856        let run = self
2857            .storage
2858            .fetch_run(run_uuid)
2859            .await
2860            .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
2861            .ok_or_else(|| Status::not_found("run not found"))?;
2862
2863        if run.status != models::RunStatus::PausedAtStep {
2864            return Err(Status::failed_precondition(
2865                "run is not paused at a breakpoint",
2866            ));
2867        }
2868
2869        let ctx_value = run.input_ctx.clone();
2870        let mut ctx_map = match ctx_value {
2871            Value::Object(map) => map,
2872            Value::Null => Map::new(),
2873            other => {
2874                return Err(internal_error(format!(
2875                    "run input context must be an object, found {other}"
2876                )))
2877            }
2878        };
2879
2880        let debug_entry = ctx_map
2881            .get_mut("debug")
2882            .and_then(Value::as_object_mut)
2883            .ok_or_else(|| internal_error("run debug context missing; cannot resume paused run"))?;
2884
2885        let paused_step = extract_active_pause_step_id(debug_entry).ok_or_else(|| {
2886            Status::failed_precondition("run is paused but no active step recorded")
2887        })?;
2888
2889        debug_entry.insert("active_pause".to_string(), Value::Null);
2890
2891        let mut tx = self
2892            .storage
2893            .pool()
2894            .begin()
2895            .await
2896            .map_err(|err| internal_error(format!("failed to begin transaction: {err}")))?;
2897
2898        self.storage
2899            .update_run_input_ctx_tx(&mut tx, run_uuid, Value::Object(ctx_map.clone()))
2900            .await
2901            .map_err(|err| internal_error(format!("failed to persist pause state: {err}")))?;
2902
2903        self.storage
2904            .set_run_status_tx(&mut tx, run_uuid, models::RunStatus::Running)
2905            .await
2906            .map_err(|err| internal_error(format!("failed to update run status: {err}")))?;
2907
2908        self.storage
2909            .enqueue_outbox_event(
2910                &mut tx,
2911                NewOutboxEvent {
2912                    run_id: run_uuid,
2913                    step_id: Some(paused_step),
2914                    kind: "run_resumed".to_string(),
2915                    payload: json!({
2916                        "run_id": run_uuid.to_string(),
2917                        "step_id": paused_step.to_string(),
2918                    }),
2919                },
2920            )
2921            .await
2922            .map_err(|err| internal_error(format!("failed to enqueue resume event: {err}")))?;
2923
2924        tx.commit()
2925            .await
2926            .map_err(|err| internal_error(format!("failed to commit resume transaction: {err}")))?;
2927
2928        if let Err(err) = self
2929            .audit
2930            .append(
2931                &actor,
2932                self.storage
2933                    .fetch_run_tenant(run_uuid)
2934                    .await
2935                    .ok()
2936                    .flatten()
2937                    .and_then(|tenant| Uuid::parse_str(&tenant).ok()),
2938                "debug.breakpoints.resume",
2939                &json!({
2940                    "run_id": run_uuid.to_string(),
2941                    "step_id": paused_step.to_string(),
2942                }),
2943            )
2944            .await
2945        {
2946            warn!(error = %err, run = %run_uuid, "failed to append audit record for resume");
2947        }
2948
2949        Ok(Response::new(runtime::ResumeRunResponse {
2950            status: runtime::RunStatus::Running as i32,
2951            resumed_step_id: paused_step.to_string(),
2952        }))
2953    }
2954
2955    async fn patch_step(
2956        &self,
2957        request: Request<runtime::PatchStepRequest>,
2958    ) -> Result<Response<runtime::PatchStepResponse>, Status> {
2959        let metadata = request.metadata().clone();
2960        let role = self.require_role(&metadata, Role::Writer).await?;
2961        let actor = Self::actor_from_metadata(&metadata, role);
2962        let req = request.into_inner();
2963
2964        let run_uuid = Uuid::parse_str(&req.run_id)
2965            .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2966        let step_uuid = Uuid::parse_str(&req.step_id)
2967            .map_err(|err| invalid_argument(format!("invalid step_id: {err}")))?;
2968
2969        let run = self
2970            .storage
2971            .fetch_run(run_uuid)
2972            .await
2973            .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
2974            .ok_or_else(|| Status::not_found("run not found"))?;
2975
2976        if run.status != models::RunStatus::PausedAtStep {
2977            return Err(Status::failed_precondition(
2978                "run must be paused before patching step",
2979            ));
2980        }
2981
2982        let ctx_value = run.input_ctx.clone();
2983        let paused_step = match ctx_value {
2984            Value::Object(ref map) => map
2985                .get("debug")
2986                .and_then(Value::as_object)
2987                .and_then(extract_active_pause_step_id),
2988            _ => None,
2989        };
2990
2991        if let Some(active_step) = paused_step {
2992            if active_step != step_uuid {
2993                return Err(Status::failed_precondition(
2994                    "only the paused step can be patched",
2995                ));
2996            }
2997        }
2998
2999        let inputs_patch_value = req.inputs_patch.as_ref().map(prost_struct_to_json);
3000        let env_override = req.env.as_ref().map(prost_struct_to_json);
3001
3002        let model_override = req.model.clone();
3003
3004        if inputs_patch_value.is_none() && env_override.is_none() && model_override.is_none() {
3005            return Err(invalid_argument(
3006                "patch_step requires inputs_patch, env, or model to be set",
3007            ));
3008        }
3009
3010        let steps = self
3011            .storage
3012            .fetch_steps_for_run(run_uuid)
3013            .await
3014            .map_err(|err| internal_error(format!("failed to fetch steps: {err}")))?;
3015
3016        let step = steps
3017            .into_iter()
3018            .find(|step| step.step_id == step_uuid)
3019            .ok_or_else(|| Status::not_found("step not found"))?;
3020
3021        let mut spec_map = match step.spec_json.clone() {
3022            Value::Object(map) => map,
3023            _ => {
3024                return Err(internal_error(
3025                    "step spec must be a JSON object to apply patches",
3026                ))
3027            }
3028        };
3029
3030        let inputs_entry = spec_map
3031            .entry("inputs".to_string())
3032            .or_insert_with(|| Value::Object(Map::new()));
3033        let inputs_map = inputs_entry
3034            .as_object_mut()
3035            .ok_or_else(|| internal_error("step inputs is not an object; cannot apply patch"))?;
3036
3037        if let Some(patch_value) = inputs_patch_value.as_ref() {
3038            let patch_obj = patch_value
3039                .as_object()
3040                .ok_or_else(|| invalid_argument("inputs_patch must be a JSON object"))?;
3041            merge_json_object(inputs_map, patch_obj);
3042        }
3043
3044        if let Some(env_value) = env_override.as_ref() {
3045            if !env_value.is_object() {
3046                return Err(invalid_argument("env override must be a JSON object"));
3047            }
3048            inputs_map.insert("env".to_string(), env_value.clone());
3049        }
3050
3051        if let Some(model) = model_override.as_ref() {
3052            inputs_map.insert("model".to_string(), Value::String(model.clone()));
3053        }
3054
3055        let updated_spec = Value::Object(spec_map.clone());
3056
3057        let mut tx = self
3058            .storage
3059            .pool()
3060            .begin()
3061            .await
3062            .map_err(|err| internal_error(format!("failed to begin transaction: {err}")))?;
3063
3064        self.storage
3065            .update_step_spec_tx(&mut tx, run_uuid, step_uuid, updated_spec.clone())
3066            .await
3067            .map_err(|err| internal_error(format!("failed to persist step patch: {err}")))?;
3068
3069        self.storage
3070            .enqueue_outbox_event(
3071                &mut tx,
3072                NewOutboxEvent {
3073                    run_id: run_uuid,
3074                    step_id: Some(step_uuid),
3075                    kind: "step_patched".to_string(),
3076                    payload: json!({
3077                        "run_id": run_uuid.to_string(),
3078                        "step_id": step_uuid.to_string(),
3079                        "inputs_patch": inputs_patch_value,
3080                        "env_override": env_override,
3081                        "model_override": model_override,
3082                        "note": req.note,
3083                    }),
3084                },
3085            )
3086            .await
3087            .map_err(|err| internal_error(format!("failed to enqueue patch event: {err}")))?;
3088
3089        tx.commit()
3090            .await
3091            .map_err(|err| internal_error(format!("failed to commit patch transaction: {err}")))?;
3092
3093        if let Err(err) = self
3094            .audit
3095            .append(
3096                &actor,
3097                self.storage
3098                    .fetch_run_tenant(run_uuid)
3099                    .await
3100                    .ok()
3101                    .flatten()
3102                    .and_then(|tenant| Uuid::parse_str(&tenant).ok()),
3103                "debug.step.patch",
3104                &json!({
3105                    "run_id": run_uuid.to_string(),
3106                    "step_id": step_uuid.to_string(),
3107                    "note": req.note,
3108                }),
3109            )
3110            .await
3111        {
3112            warn!(error = %err, run = %run_uuid, step = %step_uuid, "failed to append audit record for step patch");
3113        }
3114
3115        Ok(Response::new(runtime::PatchStepResponse {
3116            step_id: step_uuid.to_string(),
3117        }))
3118    }
3119
3120    async fn create_branch_run(
3121        &self,
3122        request: Request<runtime::CreateBranchRunRequest>,
3123    ) -> Result<Response<runtime::CreateBranchRunResponse>, Status> {
3124        let metadata = request.metadata().clone();
3125        let role = self.require_role(&metadata, Role::Writer).await?;
3126        let actor = Self::actor_from_metadata(&metadata, role);
3127        let req = request.into_inner();
3128
3129        let run_uuid = Uuid::parse_str(&req.run_id)
3130            .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
3131
3132        let from_step_spec = req.from_step.trim().to_string();
3133        if from_step_spec.is_empty() {
3134            return Err(invalid_argument("from_step is required"));
3135        }
3136
3137        let seed_mode = runtime::BranchSeedMode::try_from(req.seed_mode)
3138            .unwrap_or(runtime::BranchSeedMode::Same);
3139
3140        let run = self
3141            .storage
3142            .fetch_run(run_uuid)
3143            .await
3144            .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
3145            .ok_or_else(|| Status::not_found("run not found"))?;
3146
3147        let (inputs_value, mut labels_map) = unpack_input_ctx(&run.input_ctx);
3148
3149        let debug_entry = run
3150            .input_ctx
3151            .get("debug")
3152            .and_then(Value::as_object)
3153            .cloned()
3154            .unwrap_or_default();
3155        let slug_lookup = load_slug_map(&debug_entry);
3156        let step_ids = collect_step_ids(&run);
3157        let target_step = resolve_breakpoint(&from_step_spec, &slug_lookup, &step_ids)
3158            .map_err(|err| invalid_argument(err.to_string()))?;
3159
3160        let branch_dag = build_branch_dag(&run.dag_json, target_step)
3161            .map_err(|err| invalid_argument(err.to_string()))?;
3162
3163        let mut inputs_map = match inputs_value {
3164            Value::Object(map) => map,
3165            Value::Null => Map::new(),
3166            _ => Map::new(),
3167        };
3168
3169        let mut inputs_patch_debug: Option<Value> = None;
3170        if let Some(patch) = req.inputs_patch.as_ref() {
3171            let patch_value = prost_struct_to_json(patch);
3172            let patch_obj = patch_value
3173                .as_object()
3174                .ok_or_else(|| invalid_argument("inputs_patch must be a JSON object"))?;
3175            merge_json_object(&mut inputs_map, patch_obj);
3176            inputs_patch_debug = Some(Value::Object(patch_obj.clone()));
3177        }
3178
3179        let mut labels_patch_debug: Option<Value> = None;
3180        if let Some(labels_patch) = req.labels_patch.as_ref() {
3181            let patch_value = prost_struct_to_json(labels_patch);
3182            let patch_obj = patch_value
3183                .as_object()
3184                .ok_or_else(|| invalid_argument("labels_patch must be a JSON object"))?;
3185            for (key, value) in patch_obj {
3186                let rendered = value
3187                    .as_str()
3188                    .map(|s| s.to_string())
3189                    .unwrap_or_else(|| value.to_string());
3190                labels_map.insert(key.clone(), rendered);
3191            }
3192            labels_patch_debug = Some(Value::Object(patch_obj.clone()));
3193        }
3194
3195        labels_map.insert("branch_parent".to_string(), run_uuid.to_string());
3196        labels_map.insert("branch_from_step".to_string(), target_step.to_string());
3197
3198        let branch_seed = match seed_mode {
3199            runtime::BranchSeedMode::Same | runtime::BranchSeedMode::Unspecified => run.seed,
3200            runtime::BranchSeedMode::New => Utc::now().timestamp_nanos_opt().unwrap_or(run.seed),
3201            runtime::BranchSeedMode::Custom => req
3202                .seed
3203                .as_ref()
3204                .copied()
3205                .ok_or_else(|| invalid_argument("seed value required for custom mode"))?,
3206        };
3207
3208        let prompt_packs = collect_prompt_packs_json(&run.dag_json);
3209
3210        let mut branch_meta = Map::new();
3211        branch_meta.insert(
3212            "parent_run_id".to_string(),
3213            Value::String(run_uuid.to_string()),
3214        );
3215        branch_meta.insert(
3216            "parent_step_id".to_string(),
3217            Value::String(target_step.to_string()),
3218        );
3219        branch_meta.insert(
3220            "from_step_spec".to_string(),
3221            Value::String(from_step_spec.clone()),
3222        );
3223        branch_meta.insert(
3224            "parent_seed".to_string(),
3225            Value::Number(Number::from(run.seed)),
3226        );
3227        branch_meta.insert(
3228            "branch_seed".to_string(),
3229            Value::Number(Number::from(branch_seed)),
3230        );
3231        let seed_mode_str = match seed_mode {
3232            runtime::BranchSeedMode::Same | runtime::BranchSeedMode::Unspecified => "same",
3233            runtime::BranchSeedMode::New => "new",
3234            runtime::BranchSeedMode::Custom => "custom",
3235        };
3236        branch_meta.insert(
3237            "seed_mode".to_string(),
3238            Value::String(seed_mode_str.to_string()),
3239        );
3240        branch_meta.insert(
3241            "created_at".to_string(),
3242            Value::String(Utc::now().to_rfc3339()),
3243        );
3244        branch_meta.insert(
3245            "prompt_packs".to_string(),
3246            Value::Array(prompt_packs.into_iter().map(Value::String).collect()),
3247        );
3248        let note = req.note.trim();
3249        if !note.is_empty() {
3250            branch_meta.insert("note".to_string(), Value::String(note.to_string()));
3251        }
3252        if let Some(patch) = inputs_patch_debug {
3253            branch_meta.insert("inputs_patch".to_string(), patch);
3254        }
3255        if let Some(patch) = labels_patch_debug {
3256            branch_meta.insert("labels_patch".to_string(), patch);
3257        }
3258
3259        let mut debug_extra = Map::new();
3260        debug_extra.insert("branch".to_string(), Value::Object(branch_meta));
3261
3262        let submission = SubmitRunSpec {
3263            dag_json: branch_dag,
3264            inputs: Value::Object(inputs_map.clone()),
3265            labels: labels_map.clone(),
3266            seed: branch_seed,
3267            traceparent: None,
3268            tracestate: None,
3269            baggage: None,
3270            debug: Some(Value::Object(debug_extra.clone())),
3271            breakpoints: Vec::new(),
3272        };
3273
3274        let outcome = submit_run_impl(
3275            self.storage.as_ref(),
3276            self.audit.as_ref(),
3277            submission,
3278            None,
3279            &actor,
3280        )
3281        .await
3282        .map_err(|err| match err {
3283            SubmitRunError::Invalid(message) => invalid_argument(message),
3284            SubmitRunError::Internal(message) => internal_error(message),
3285        })?;
3286
3287        let new_run_uuid = match outcome {
3288            SubmitRunOutcome::Created(run_id) | SubmitRunOutcome::Reused(run_id) => run_id,
3289        };
3290
3291        let mut tx = self.storage.pool().begin().await.map_err(|err| {
3292            internal_error(format!("failed to begin branch event transaction: {err}"))
3293        })?;
3294
3295        self.storage
3296            .enqueue_outbox_event(
3297                &mut tx,
3298                NewOutboxEvent {
3299                    run_id: run_uuid,
3300                    step_id: Some(target_step),
3301                    kind: "run_branch_created".to_string(),
3302                    payload: json!({
3303                        "run_id": run_uuid.to_string(),
3304                        "child_run_id": new_run_uuid.to_string(),
3305                        "from_step_spec": from_step_spec,
3306                        "seed": branch_seed,
3307                    }),
3308                },
3309            )
3310            .await
3311            .map_err(|err| internal_error(format!("failed to enqueue branch event: {err}")))?;
3312
3313        tx.commit().await.map_err(|err| {
3314            internal_error(format!("failed to commit branch event transaction: {err}"))
3315        })?;
3316
3317        if let Err(err) = self
3318            .audit
3319            .append(
3320                &actor,
3321                self.storage
3322                    .fetch_run_tenant(run_uuid)
3323                    .await
3324                    .ok()
3325                    .flatten()
3326                    .and_then(|tenant| Uuid::parse_str(&tenant).ok()),
3327                "debug.branch.create",
3328                &json!({
3329                    "run_id": run_uuid.to_string(),
3330                    "child_run_id": new_run_uuid.to_string(),
3331                    "step_id": target_step.to_string(),
3332                }),
3333            )
3334            .await
3335        {
3336            warn!(error = %err, run = %run_uuid, "failed to append audit record for branch");
3337        }
3338
3339        Ok(Response::new(runtime::CreateBranchRunResponse {
3340            run_id: new_run_uuid.to_string(),
3341        }))
3342    }
3343
3344    async fn get_attestations(
3345        &self,
3346        request: Request<runtime::GetAttestationsRequest>,
3347    ) -> Result<Response<runtime::GetAttestationsResponse>, Status> {
3348        let metadata = request.metadata().clone();
3349        self.require_role(&metadata, Role::Reader).await?;
3350        let req = request.into_inner();
3351        let mut seen: HashSet<Uuid> = HashSet::new();
3352        let mut results = Vec::new();
3353
3354        for raw in req.attestation_ids {
3355            let trimmed = raw.trim();
3356            if trimmed.is_empty() {
3357                continue;
3358            }
3359            let id = Uuid::parse_str(trimmed).map_err(|err| {
3360                invalid_argument(format!("invalid attestation id '{trimmed}': {err}"))
3361            })?;
3362            if !seen.insert(id) {
3363                continue;
3364            }
3365            match self.attestation_vault.fetch(id).await {
3366                Ok(Some(attestation)) => {
3367                    results.push(attestation_to_proto(attestation)?);
3368                }
3369                Ok(None) => {
3370                    // Ignore missing entries so clients can compare counts.
3371                }
3372                Err(err) => {
3373                    return Err(internal_error(format!(
3374                        "failed to load attestation {id}: {err}"
3375                    )));
3376                }
3377            }
3378        }
3379
3380        Ok(Response::new(runtime::GetAttestationsResponse {
3381            attestations: results,
3382        }))
3383    }
3384
3385    async fn sync_eval_pack(
3386        &self,
3387        request: Request<runtime::SyncEvalPackRequest>,
3388    ) -> Result<Response<runtime::SyncEvalPackResponse>, Status> {
3389        let metadata = request.metadata().clone();
3390        self.require_role(&metadata, Role::Writer).await?;
3391
3392        let req = request.into_inner();
3393        let pack = req.pack.trim();
3394        if pack.is_empty() {
3395            return Err(invalid_argument("pack is required"));
3396        }
3397        let pack_name = pack.to_string();
3398
3399        let mut scenario_ids = Vec::with_capacity(req.scenarios.len());
3400        for scenario in req.scenarios {
3401            let slug = scenario.slug.trim();
3402            if slug.is_empty() {
3403                return Err(invalid_argument("scenario slug cannot be empty"));
3404            }
3405            let run_spec_struct = scenario
3406                .run_spec
3407                .ok_or_else(|| invalid_argument("scenario run_spec must be provided"))?;
3408            let expected_struct = scenario
3409                .expected
3410                .ok_or_else(|| invalid_argument("scenario expected payload must be provided"))?;
3411
3412            // prost_struct_to_json yields a serde_json::Value directly.
3413            let run_spec_json = prost_struct_to_json(&run_spec_struct);
3414            let expected_json = prost_struct_to_json(&expected_struct);
3415
3416            let title = scenario.title.trim();
3417            let description = scenario.description.trim();
3418
3419            let mut spec_map = Map::new();
3420            spec_map.insert("pack".to_string(), Value::String(pack_name.clone()));
3421            spec_map.insert("slug".to_string(), Value::String(slug.to_string()));
3422            if !title.is_empty() {
3423                spec_map.insert("title".to_string(), Value::String(title.to_string()));
3424            }
3425            if !description.is_empty() {
3426                spec_map.insert(
3427                    "description".to_string(),
3428                    Value::String(description.to_string()),
3429                );
3430            }
3431            spec_map.insert("run_spec".to_string(), run_spec_json);
3432            spec_map.insert("expected".to_string(), expected_json);
3433            let spec_value = Value::Object(spec_map);
3434
3435            let scenario_name = format!("{pack}/{slug}");
3436            let scenario_id = self
3437                .storage
3438                .upsert_eval_scenario(
3439                    &scenario_name,
3440                    if description.is_empty() {
3441                        None
3442                    } else {
3443                        Some(description)
3444                    },
3445                    spec_value,
3446                )
3447                .await
3448                .map_err(|err| {
3449                    internal_error(format!(
3450                        "failed to upsert eval scenario '{scenario_name}': {err}"
3451                    ))
3452                })?;
3453            scenario_ids.push(scenario_id.to_string());
3454        }
3455
3456        Ok(Response::new(runtime::SyncEvalPackResponse {
3457            scenario_ids,
3458        }))
3459    }
3460
3461    async fn list_eval_scenarios(
3462        &self,
3463        request: Request<runtime::ListEvalScenariosRequest>,
3464    ) -> Result<Response<runtime::ListEvalScenariosResponse>, Status> {
3465        let metadata = request.metadata().clone();
3466        self.require_role(&metadata, Role::Reader).await?;
3467
3468        let scenarios = self
3469            .storage
3470            .list_eval_scenarios()
3471            .await
3472            .map_err(|err| internal_error(format!("failed to list eval scenarios: {err}")))?;
3473
3474        let mut proto_scenarios = Vec::with_capacity(scenarios.len());
3475        for scenario in scenarios {
3476            let spec = parse_eval_spec(&scenario.spec).map_err(|err| {
3477                internal_error(format!(
3478                    "invalid eval scenario '{}' spec: {err}",
3479                    scenario.name
3480                ))
3481            })?;
3482            let history_models = self
3483                .storage
3484                .list_eval_results(scenario.scenario_id, 20)
3485                .await
3486                .map_err(|err| internal_error(format!("failed to fetch eval results: {err}")))?;
3487            let mut history_proto = Vec::with_capacity(history_models.len());
3488            for result in history_models {
3489                history_proto.push(build_eval_result_proto(result)?);
3490            }
3491            proto_scenarios.push(build_eval_scenario_proto(&scenario, &spec, history_proto)?);
3492        }
3493
3494        Ok(Response::new(runtime::ListEvalScenariosResponse {
3495            scenarios: proto_scenarios,
3496        }))
3497    }
3498
3499    async fn record_eval_result(
3500        &self,
3501        request: Request<runtime::RecordEvalResultRequest>,
3502    ) -> Result<Response<runtime::RecordEvalResultResponse>, Status> {
3503        let metadata = request.metadata().clone();
3504        self.require_role(&metadata, Role::Writer).await?;
3505
3506        let req = request.into_inner();
3507        let pack = req.pack.trim();
3508        let slug = req.slug.trim();
3509        if pack.is_empty() {
3510            return Err(invalid_argument("pack is required"));
3511        }
3512        if slug.is_empty() {
3513            return Err(invalid_argument("slug is required"));
3514        }
3515
3516        let scenario_name = format!("{pack}/{slug}");
3517        let scenario = self
3518            .storage
3519            .fetch_eval_scenario_by_name(&scenario_name)
3520            .await
3521            .map_err(|err| internal_error(format!("failed to fetch eval scenario: {err}")))?
3522            .ok_or_else(|| {
3523                Status::not_found(format!("eval scenario '{scenario_name}' not found"))
3524            })?;
3525
3526        let spec = parse_eval_spec(&scenario.spec).map_err(|err| {
3527            internal_error(format!(
3528                "invalid eval scenario '{}' spec: {err}",
3529                scenario.name
3530            ))
3531        })?;
3532
3533        let run_uuid = Uuid::parse_str(&req.run_id)
3534            .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
3535        let run = self
3536            .storage
3537            .fetch_run(run_uuid)
3538            .await
3539            .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
3540            .ok_or_else(|| Status::not_found("run not found"))?;
3541        let steps = self
3542            .storage
3543            .fetch_steps_for_run(run_uuid)
3544            .await
3545            .map_err(|err| internal_error(format!("failed to fetch steps: {err}")))?;
3546        let last_event = self
3547            .storage
3548            .fetch_last_outbox_event(run_uuid)
3549            .await
3550            .map_err(|err| internal_error(format!("failed to fetch run events: {err}")))?;
3551
3552        let previous = self
3553            .storage
3554            .latest_eval_result(scenario.scenario_id)
3555            .await
3556            .map_err(|err| {
3557                internal_error(format!("failed to fetch previous eval result: {err}"))
3558            })?;
3559        let previous_metrics = previous
3560            .as_ref()
3561            .filter(|result| result.run_id != run_uuid)
3562            .map(|result| &result.metrics);
3563
3564        let commit = req.commit.trim();
3565        let commit = if commit.is_empty() {
3566            None
3567        } else {
3568            Some(commit.to_string())
3569        };
3570
3571        let (outcome, metrics) = compute_eval_metrics(
3572            &spec,
3573            &run,
3574            &steps,
3575            last_event.as_ref().map(|event| event.created_at),
3576            previous_metrics,
3577            commit.clone(),
3578        )
3579        .map_err(|err| internal_error(format!("failed to compute eval metrics: {err}")))?;
3580
3581        let stored = self
3582            .storage
3583            .insert_eval_result(scenario.scenario_id, run_uuid, &outcome, metrics)
3584            .await
3585            .map_err(|err| internal_error(format!("failed to persist eval result: {err}")))?;
3586
3587        let proto = build_eval_result_proto(stored)?;
3588        Ok(Response::new(runtime::RecordEvalResultResponse {
3589            result: Some(proto),
3590        }))
3591    }
3592
3593    async fn check_change_gate(
3594        &self,
3595        request: Request<runtime::CheckChangeGateRequest>,
3596    ) -> Result<Response<runtime::CheckChangeGateResponse>, Status> {
3597        let metadata = request.metadata().clone();
3598        let role = self.require_role(&metadata, Role::Writer).await?;
3599        let req = request.into_inner();
3600
3601        let change_id = req.change_id.trim();
3602        if change_id.is_empty() {
3603            return Err(invalid_argument("change_id is required"));
3604        }
3605
3606        let revision = req.revision.trim();
3607        let revision = if revision.is_empty() {
3608            None
3609        } else {
3610            Some(revision.to_string())
3611        };
3612
3613        let actor_header = Self::actor_from_metadata(&metadata, role);
3614        let actor = if req.actor.trim().is_empty() {
3615            actor_header
3616        } else {
3617            req.actor.trim().to_string()
3618        };
3619
3620        let diffs = req
3621            .diffs
3622            .into_iter()
3623            .map(|diff| GateDiff {
3624                path: diff.path,
3625                lines_added: diff.lines_added,
3626                lines_deleted: diff.lines_deleted,
3627                novelty_score: diff.novelty_score,
3628                risk_tags: diff.risk_tags,
3629                component: if diff.component.is_empty() {
3630                    None
3631                } else {
3632                    Some(diff.component)
3633                },
3634            })
3635            .collect();
3636
3637        let coverage = req
3638            .coverage
3639            .map(|report| GateCoverageReport {
3640                overall_ratio: report.overall_ratio,
3641                minimum_ratio: report.minimum_ratio,
3642                components: report
3643                    .components
3644                    .into_iter()
3645                    .map(|component| GateCoverageComponent {
3646                        name: component.name,
3647                        coverage_ratio: component.coverage_ratio,
3648                        required_ratio: component.required_ratio,
3649                    })
3650                    .collect(),
3651            })
3652            .unwrap_or_default();
3653
3654        let evals = req
3655            .evals
3656            .into_iter()
3657            .map(|metric| GateEvalMetric {
3658                name: metric.name,
3659                slug: if metric.slug.is_empty() {
3660                    None
3661                } else {
3662                    Some(metric.slug)
3663                },
3664                score: metric.score,
3665                threshold: metric.threshold,
3666                direction: match runtime::MetricDirection::try_from(metric.direction) {
3667                    Ok(runtime::MetricDirection::LowerIsBetter) => {
3668                        GateMetricDirection::LowerIsBetter
3669                    }
3670                    _ => GateMetricDirection::HigherIsBetter,
3671                },
3672                weight: metric.weight,
3673            })
3674            .collect();
3675
3676        let budgets = req
3677            .budgets
3678            .into_iter()
3679            .map(|budget| GateBudgetSignal {
3680                name: budget.name,
3681                actual: budget.actual,
3682                limit: budget.limit,
3683            })
3684            .collect();
3685
3686        let telemetry_value = struct_to_value(req.telemetry);
3687        let metadata_value = struct_to_value(req.metadata);
3688
3689        let gate_request = GateRequest {
3690            change_id: change_id.to_string(),
3691            gate_id: None,
3692            revision: revision.clone(),
3693            diffs,
3694            coverage,
3695            evals,
3696            budgets,
3697            telemetry: telemetry_value.clone(),
3698            metadata: metadata_value.clone(),
3699            actor: Some(actor.clone()),
3700        };
3701
3702        let mut decision = self
3703            .changeops
3704            .evaluate(&gate_request)
3705            .map_err(|err| invalid_argument(err.to_string()))?;
3706
3707        let gate_id = Uuid::new_v4();
3708        let mut stored_request = gate_request.clone();
3709        stored_request.gate_id = Some(gate_id);
3710
3711        let decision_payload = decision.to_value(&stored_request);
3712        let decision_bytes = serde_json::to_vec(&decision_payload).map_err(|err| {
3713            internal_error(format!("failed to serialise change gate decision: {err}"))
3714        })?;
3715
3716        let artifact_meta = self
3717            .storage
3718            .artifacts()
3719            .put(
3720                Bytes::from(decision_bytes),
3721                "application/json",
3722                json!({
3723                    "kind": "change_gate_decision",
3724                    "change_id": change_id,
3725                    "effect": decision.effect.as_str(),
3726                    "gate_id": gate_id.to_string(),
3727                }),
3728            )
3729            .await
3730            .map_err(|err| {
3731                internal_error(format!("failed to persist change gate artifact: {err}"))
3732            })?;
3733
3734        let mut attestation_set: HashSet<Uuid> =
3735            attestation_ids_from_value(&stored_request.telemetry)
3736                .into_iter()
3737                .collect();
3738        attestation_set.extend(attestation_ids_from_value(&decision_payload));
3739        let attestation_refs: Vec<Uuid> = attestation_set.into_iter().collect();
3740
3741        if !attestation_refs.is_empty() && self.transparency_scope.includes_gates() {
3742            let artifact_sha = hex::encode(artifact_meta.sha256);
3743            if self.transparency_writer_enabled {
3744                let payload =
3745                    models::TransparencyJobPayload::ReleaseBundle(models::ReleaseBundlePayload {
3746                        change_id: change_id.to_string(),
3747                        gate_id,
3748                        artifact_sha256: artifact_sha.clone(),
3749                        attestation_ids: attestation_refs.clone(),
3750                        metadata: decision.metadata.clone(),
3751                    });
3752                match self.storage.enqueue_transparency_job(payload).await {
3753                    Ok(job_id) => {
3754                        let queued = json!({
3755                            "status": "queued",
3756                            "job_id": job_id,
3757                            "artifact_sha256": artifact_sha,
3758                            "attestation_ids": attestation_refs.iter().map(|id| id.to_string()).collect::<Vec<_>>(),
3759                        });
3760                        if let Value::Object(mut meta_obj) = decision.metadata.take() {
3761                            meta_obj.insert("scitt_entry".to_string(), queued);
3762                            decision.metadata = Value::Object(meta_obj);
3763                        } else {
3764                            decision.metadata = json!({ "scitt_entry": queued });
3765                        }
3766                    }
3767                    Err(err) => {
3768                        warn!(
3769                            change = change_id,
3770                            gate = %gate_id,
3771                            error = %err,
3772                            "failed to enqueue transparency job; falling back to synchronous publish"
3773                        );
3774                        self.publish_scitt_entry(
3775                            change_id,
3776                            gate_id,
3777                            &artifact_sha,
3778                            &attestation_refs,
3779                            &mut decision.metadata,
3780                        )
3781                        .await?;
3782                    }
3783                }
3784            } else {
3785                self.publish_scitt_entry(
3786                    change_id,
3787                    gate_id,
3788                    &artifact_sha,
3789                    &attestation_refs,
3790                    &mut decision.metadata,
3791                )
3792                .await?;
3793            }
3794        }
3795
3796        let stored_gate = self
3797            .storage
3798            .insert_change_gate(models::NewChangeGate {
3799                gate_id,
3800                change_id: change_id.to_string(),
3801                revision: revision.clone(),
3802                effect: decision.effect.as_str().to_string(),
3803                reasons: serde_json::to_value(&decision.reasons).map_err(|err| {
3804                    internal_error(format!("failed to serialise change gate reasons: {err}"))
3805                })?,
3806                followups: serde_json::to_value(&decision.followups).map_err(|err| {
3807                    internal_error(format!("failed to serialise change gate followups: {err}"))
3808                })?,
3809                scorecard: serde_json::to_value(&decision.scorecard).map_err(|err| {
3810                    internal_error(format!("failed to serialise change gate scorecard: {err}"))
3811                })?,
3812                decided_at: decision.decided_at,
3813                decided_by: Some(actor.clone()),
3814                metadata: decision.metadata.clone(),
3815                input: serde_json::to_value(&stored_request).map_err(|err| {
3816                    internal_error(format!("failed to serialise change gate request: {err}"))
3817                })?,
3818                telemetry: stored_request.telemetry.clone(),
3819                artifact_sha256: Some(artifact_meta.sha256),
3820            })
3821            .await
3822            .map_err(|err| {
3823                internal_error(format!("failed to persist change gate decision: {err}"))
3824            })?;
3825
3826        let audit_payload = decision_payload.clone();
3827        self.audit
3828            .append(&actor, None, "change_gate.check", &audit_payload)
3829            .await
3830            .map_err(|err| {
3831                internal_error(format!(
3832                    "failed to append audit trail for change gate: {err}"
3833                ))
3834            })?;
3835
3836        tracing::info!(
3837            target: "fleetforge.changeops",
3838            gate = %gate_id,
3839            change = %change_id,
3840            effect = decision.effect.as_str(),
3841            reasons = ?decision.reasons,
3842            "change gate evaluated"
3843        );
3844
3845        let proto_decision = build_proto_change_gate(
3846            stored_gate.gate_id,
3847            change_id,
3848            revision.as_deref(),
3849            &decision,
3850            &stored_gate.telemetry,
3851            &[],
3852        )?;
3853
3854        Ok(Response::new(runtime::CheckChangeGateResponse {
3855            decision: Some(proto_decision),
3856        }))
3857    }
3858
3859    async fn record_gate_followup(
3860        &self,
3861        request: Request<runtime::RecordGateFollowupRequest>,
3862    ) -> Result<Response<runtime::RecordGateFollowupResponse>, Status> {
3863        let metadata = request.metadata().clone();
3864        let role = self.require_role(&metadata, Role::Writer).await?;
3865        let req = request.into_inner();
3866
3867        let gate_id = Uuid::parse_str(&req.gate_id)
3868            .map_err(|err| invalid_argument(format!("invalid gate_id: {err}")))?;
3869
3870        let gate = self
3871            .storage
3872            .fetch_change_gate(gate_id)
3873            .await
3874            .map_err(|err| internal_error(format!("failed to fetch change gate: {err}")))?
3875            .ok_or_else(|| Status::not_found("change gate not found"))?;
3876
3877        let actor_header = Self::actor_from_metadata(&metadata, role);
3878        let actor = if req.actor.trim().is_empty() {
3879            actor_header
3880        } else {
3881            req.actor.trim().to_string()
3882        };
3883
3884        let outcome = runtime::FollowupOutcome::try_from(req.outcome)
3885            .unwrap_or(runtime::FollowupOutcome::Approved);
3886        let outcome_gate = proto_followup_outcome(outcome);
3887
3888        let followup_id = Uuid::new_v4();
3889        let note = req.note;
3890        let details_value = struct_to_value(req.details);
3891
3892        let stored_followup = self
3893            .storage
3894            .insert_change_gate_followup(models::NewChangeGateFollowup {
3895                followup_id,
3896                gate_id,
3897                actor: actor.clone(),
3898                outcome: outcome_gate.as_str().to_string(),
3899                note: note.clone(),
3900                details: details_value.clone(),
3901            })
3902            .await
3903            .map_err(|err| {
3904                internal_error(format!("failed to persist change gate follow-up: {err}"))
3905            })?;
3906
3907        let record_payload = ChangeFollowupRecord {
3908            gate_id,
3909            actor: actor.clone(),
3910            outcome: outcome_gate,
3911            note: note.clone(),
3912            details: stored_followup.details.clone(),
3913            recorded_at: stored_followup.recorded_at,
3914        };
3915
3916        self.audit
3917            .append(
3918                &actor,
3919                None,
3920                "change_gate.followup",
3921                &record_payload.to_value(),
3922            )
3923            .await
3924            .map_err(|err| {
3925                internal_error(format!(
3926                    "failed to append change gate follow-up audit: {err}"
3927                ))
3928            })?;
3929
3930        tracing::info!(
3931            target: "fleetforge.changeops",
3932            gate = %gate_id,
3933            followup = %followup_id,
3934            outcome = %stored_followup.outcome,
3935            "change gate follow-up recorded"
3936        );
3937
3938        let proto_followup = gate_followup_record_to_proto(&stored_followup);
3939        Ok(Response::new(runtime::RecordGateFollowupResponse {
3940            followup: Some(proto_followup),
3941        }))
3942    }
3943
3944    async fn list_change_gates(
3945        &self,
3946        request: Request<runtime::ListChangeGatesRequest>,
3947    ) -> Result<Response<runtime::ListChangeGatesResponse>, Status> {
3948        let metadata = request.metadata().clone();
3949        self.require_role(&metadata, Role::Reader).await?;
3950        let req = request.into_inner();
3951
3952        let change_id = req.change_id.trim();
3953        if change_id.is_empty() {
3954            return Err(invalid_argument("change_id is required"));
3955        }
3956
3957        let limit = if req.limit == 0 { 5 } else { req.limit.min(50) } as i64;
3958
3959        let gates = self
3960            .storage
3961            .list_change_gates_for_change(change_id, limit)
3962            .await
3963            .map_err(|err| {
3964                internal_error(format!("failed to fetch change gate decisions: {err}"))
3965            })?;
3966
3967        let mut decisions = Vec::with_capacity(gates.len());
3968        for gate in gates {
3969            let followups = self
3970                .storage
3971                .fetch_change_gate_followups(gate.gate_id)
3972                .await
3973                .map_err(|err| {
3974                    internal_error(format!("failed to fetch change gate follow-ups: {err}"))
3975                })?;
3976
3977            let effect = gate_effect_from_str(&gate.effect);
3978            let reasons = parse_reasons(&gate.reasons);
3979            let followup_actions = parse_followup_actions(&gate.followups);
3980            let scorecard = parse_scorecard(&gate.scorecard);
3981
3982            let gate_decision = GateDecision {
3983                effect,
3984                reasons,
3985                followups: followup_actions,
3986                scorecard,
3987                decided_at: gate.decided_at,
3988                metadata: gate.metadata.clone(),
3989            };
3990
3991            let proto_decision = build_proto_change_gate(
3992                gate.gate_id,
3993                &gate.change_id,
3994                gate.revision.as_deref(),
3995                &gate_decision,
3996                &gate.telemetry,
3997                &followups,
3998            )?;
3999            decisions.push(proto_decision);
4000        }
4001
4002        Ok(Response::new(runtime::ListChangeGatesResponse {
4003            decisions,
4004        }))
4005    }
4006}
4007
4008/// Starts the API servers (gRPC + HTTP) backed by the provided storage handle.
4009#[instrument(skip_all)]
4010pub async fn start_api_server(
4011    storage: Arc<Storage>,
4012    audit: Arc<Audit>,
4013    metadata: Arc<RuntimeMetadata>,
4014) -> Result<()> {
4015    let auth = Arc::new(AuthHandle::from_env().await?);
4016
4017    let endpoints = resolve_endpoint_config()?;
4018    let cors_config = cors_config_from_env();
4019    let transparency = transparency_log_from_env(Arc::clone(&storage))?;
4020    let transparency_scope = transparency_scope_from_env();
4021    let transparency_writer_enabled = transparency_writer_enabled_flag();
4022    let writer_interval = transparency_writer_interval();
4023
4024    if transparency_writer_enabled && !matches!(transparency_scope, TransparencyScope::Disabled) {
4025        info!(
4026            "transparency writer enabled (interval={}s)",
4027            writer_interval.as_secs()
4028        );
4029        let writer_storage = Arc::clone(&storage);
4030        let writer_log = Arc::clone(&transparency);
4031        tokio::spawn(async move {
4032            let writer = TransparencyWriter::new(writer_storage, writer_log, writer_interval);
4033            writer.run().await;
4034        });
4035    }
4036
4037    let cors_summary = cors_config
4038        .as_ref()
4039        .map(|cfg| {
4040            if cfg.display.is_empty() {
4041                "[]".to_string()
4042            } else {
4043                cfg.display.join(", ")
4044            }
4045        })
4046        .unwrap_or_else(|| "disabled".to_string());
4047
4048    info!(
4049        grpc_endpoint = %format!("http://{}", endpoints.grpc_addr),
4050        http_endpoint = %format!("http://{}", endpoints.http_addr),
4051        cors_allowlist = %cors_summary,
4052        "runtime API configuration ready"
4053    );
4054
4055    if endpoints.combined {
4056        run_combined_server(
4057            endpoints.grpc_addr,
4058            storage,
4059            auth,
4060            audit,
4061            transparency,
4062            transparency_writer_enabled,
4063            transparency_scope,
4064            metadata,
4065            cors_config,
4066        )
4067        .await
4068    } else {
4069        let grpc_storage = Arc::clone(&storage);
4070        let http_storage = Arc::clone(&storage);
4071        let http_auth = Arc::clone(&auth);
4072        let grpc_transparency = Arc::clone(&transparency);
4073        let version_meta = Arc::clone(&metadata);
4074        tokio::try_join!(
4075            run_grpc_server(
4076                endpoints.grpc_addr,
4077                grpc_storage,
4078                Arc::clone(&auth),
4079                Arc::clone(&audit),
4080                grpc_transparency,
4081                transparency_writer_enabled,
4082                transparency_scope,
4083                cors_config.clone(),
4084            ),
4085            run_http_server(
4086                endpoints.http_addr,
4087                http_storage,
4088                http_auth,
4089                Arc::clone(&audit),
4090                version_meta,
4091                cors_config,
4092            ),
4093        )?;
4094        Ok(())
4095    }
4096}
4097
4098#[derive(Clone, Copy, Debug)]
4099struct EndpointConfig {
4100    grpc_addr: SocketAddr,
4101    http_addr: SocketAddr,
4102    combined: bool,
4103}
4104
4105fn resolve_endpoint_config() -> Result<EndpointConfig> {
4106    if let Ok(port_raw) = std::env::var("PORT") {
4107        let port: u16 = port_raw.parse().context("invalid PORT value")?;
4108        let addr = SocketAddr::from(([0, 0, 0, 0], port));
4109        return Ok(EndpointConfig {
4110            grpc_addr: addr,
4111            http_addr: addr,
4112            combined: true,
4113        });
4114    }
4115
4116    let grpc_addr: SocketAddr = std::env::var("FLEETFORGE_API_ADDR")
4117        .unwrap_or_else(|_| "0.0.0.0:50051".to_string())
4118        .parse()
4119        .context("invalid FLEETFORGE_API_ADDR")?;
4120    let http_addr: SocketAddr = std::env::var("FLEETFORGE_HTTP_ADDR")
4121        .unwrap_or_else(|_| "0.0.0.0:8080".to_string())
4122        .parse()
4123        .context("invalid FLEETFORGE_HTTP_ADDR")?;
4124
4125    Ok(EndpointConfig {
4126        grpc_addr,
4127        http_addr,
4128        combined: grpc_addr == http_addr,
4129    })
4130}
4131
4132#[instrument(skip_all)]
4133async fn run_combined_server(
4134    addr: SocketAddr,
4135    storage: Arc<Storage>,
4136    auth: Arc<AuthHandle>,
4137    audit: Arc<Audit>,
4138    transparency: Arc<dyn TransparencyLog>,
4139    transparency_writer_enabled: bool,
4140    transparency_scope: TransparencyScope,
4141    metadata: Arc<RuntimeMetadata>,
4142    cors: Option<CorsConfig>,
4143) -> Result<()> {
4144    let listener = TcpListener::bind(addr)
4145        .await
4146        .with_context(|| format!("failed to bind listener on {addr}"))?;
4147
4148    let attestation_vault: Arc<dyn AttestationVault> = Arc::new(ObjectStoreAttestationVault::new(
4149        storage.object_store(),
4150        storage.pool().clone(),
4151    ));
4152    let runtime_api = RuntimeApiService::new(
4153        Arc::clone(&storage),
4154        Arc::clone(&auth),
4155        Arc::clone(&audit),
4156        Arc::clone(&attestation_vault),
4157        Arc::clone(&transparency),
4158        transparency_writer_enabled,
4159        transparency_scope,
4160    );
4161    let tap_api = TapApiService::new(Arc::clone(&storage), Arc::clone(&auth));
4162
4163    let (mut health_reporter, health_service) = health_reporter();
4164    health_reporter
4165        .set_service_status(
4166            "fleetforge.runtime.v1.RuntimeService",
4167            tonic_health::ServingStatus::Serving,
4168        )
4169        .await; // API returns () so there is nothing to convert via map_err.
4170    health_reporter
4171        .set_service_status(
4172            "fleetforge.tap.v2.TapService",
4173            tonic_health::ServingStatus::Serving,
4174        )
4175        .await;
4176    health_reporter
4177        .set_service_status("", tonic_health::ServingStatus::Serving)
4178        .await;
4179
4180    let runtime_service = RuntimeServiceServer::new(runtime_api);
4181    let runtime_service = tonic_web::enable(runtime_service);
4182    let tap_service = TapServiceServer::new(tap_api);
4183    let tap_service = tonic_web::enable(tap_service);
4184
4185    let reflection = ReflectionBuilder::configure()
4186        .register_encoded_file_descriptor_set(runtime::FILE_DESCRIPTOR_SET)
4187        .build()
4188        .context("failed to build reflection service")?;
4189
4190    let cors_layer = cors.as_ref().map(|cfg| cfg.to_layer());
4191    let http_router = {
4192        let router = http::router(
4193            Arc::clone(&storage),
4194            Arc::clone(&auth),
4195            Arc::clone(&audit),
4196            Arc::clone(&metadata),
4197        );
4198        match cors_layer.clone() {
4199            Some(layer) => router.layer(layer.clone()),
4200            None => router,
4201        }
4202    };
4203    let mut builder = Server::builder().accept_http1(true);
4204
4205    let grpc_service = builder
4206        .add_service(health_service)
4207        .add_service(reflection)
4208        .add_service(runtime_service)
4209        .add_service(tap_service)
4210        .into_service();
4211
4212    let grpc_service = BoxCloneService::new(grpc_service);
4213
4214    info!(%addr, "starting runtime API server (single listener)");
4215
4216    let make_service = make_service_fn(move |_conn| {
4217        let http_router = http_router.clone();
4218        let grpc_service = grpc_service.clone();
4219
4220        async move {
4221            Ok::<_, Infallible>(service_fn(move |req: HyperRequest<Body>| {
4222                let http_router = http_router.clone();
4223                let mut grpc_service = grpc_service.clone();
4224
4225                async move {
4226                    if is_grpc_request(&req) {
4227                        tower::Service::call(&mut grpc_service, req).await
4228                    } else {
4229                        let router = http_router.clone();
4230                        let response = match router.oneshot(req).await {
4231                            Ok(resp) => resp,
4232                            Err(err) => match err {},
4233                        }
4234                        .map(|body| {
4235                            body.map_err(|err| Status::internal(err.to_string()))
4236                                .boxed_unsync()
4237                        });
4238                        Ok::<_, Box<dyn std::error::Error + Send + Sync>>(response)
4239                    }
4240                }
4241            }))
4242        }
4243    });
4244
4245    let std_listener = listener
4246        .into_std()
4247        .context("failed to convert listener for hyper server")?;
4248    HyperServer::from_tcp(std_listener)
4249        .context("failed to build runtime API server")?
4250        .serve(make_service)
4251        .await
4252        .context("runtime API server exited unexpectedly")
4253}
4254
4255fn is_grpc_request(req: &HyperRequest<Body>) -> bool {
4256    if let Some(content_type) = req
4257        .headers()
4258        .get(header::CONTENT_TYPE)
4259        .and_then(|value| value.to_str().ok())
4260    {
4261        let lower = content_type.to_ascii_lowercase();
4262        if lower.starts_with("application/grpc")
4263            || lower.starts_with("application/grpc-web")
4264            || lower.starts_with("application/connect+")
4265        {
4266            return true;
4267        }
4268    }
4269
4270    let path = req.uri().path();
4271    path.starts_with("/fleetforge.runtime.v1.")
4272        || path.starts_with("/fleetforge.tap.v2.")
4273        || path.starts_with("/grpc.health.v1.")
4274        || path.starts_with("/grpc.reflection.v1alpha.")
4275}
4276
4277#[instrument(skip_all)]
4278async fn run_grpc_server(
4279    addr: SocketAddr,
4280    storage: Arc<Storage>,
4281    auth: Arc<AuthHandle>,
4282    audit: Arc<Audit>,
4283    transparency: Arc<dyn TransparencyLog>,
4284    transparency_writer_enabled: bool,
4285    transparency_scope: TransparencyScope,
4286    cors: Option<CorsConfig>,
4287) -> Result<()> {
4288    let attestation_vault: Arc<dyn AttestationVault> = Arc::new(ObjectStoreAttestationVault::new(
4289        storage.object_store(),
4290        storage.pool().clone(),
4291    ));
4292    let runtime_api = RuntimeApiService::new(
4293        Arc::clone(&storage),
4294        Arc::clone(&auth),
4295        Arc::clone(&audit),
4296        Arc::clone(&attestation_vault),
4297        transparency,
4298        transparency_writer_enabled,
4299        transparency_scope,
4300    );
4301    let tap_api = TapApiService::new(storage, auth);
4302
4303    let (mut health_reporter, health_service) = health_reporter();
4304    let reflection = ReflectionBuilder::configure()
4305        .register_encoded_file_descriptor_set(runtime::FILE_DESCRIPTOR_SET)
4306        .build()
4307        .context("failed to build reflection service")?;
4308
4309    health_reporter
4310        .set_service_status(
4311            "fleetforge.runtime.v1.RuntimeService",
4312            tonic_health::ServingStatus::Serving,
4313        )
4314        .await; // set_service_status returns ()
4315    health_reporter
4316        .set_service_status(
4317            "fleetforge.tap.v2.TapService",
4318            tonic_health::ServingStatus::Serving,
4319        )
4320        .await;
4321    health_reporter
4322        .set_service_status("", tonic_health::ServingStatus::Serving)
4323        .await;
4324
4325    info!(%addr, "starting runtime gRPC API server");
4326
4327    let runtime_service = RuntimeServiceServer::new(runtime_api);
4328    let runtime_service = tonic_web::enable(runtime_service);
4329    let tap_service = TapServiceServer::new(tap_api);
4330    let tap_service = tonic_web::enable(tap_service);
4331    let cors_layer = cors.as_ref().map(|cfg| cfg.to_layer());
4332    Server::builder()
4333        .accept_http1(true)
4334        .add_service(health_service)
4335        .add_service(reflection)
4336        .add_service(runtime_service)
4337        .add_service(tap_service)
4338        .serve(addr)
4339        .await
4340        .context("runtime gRPC API server exited unexpectedly")
4341}
4342
4343#[instrument(skip_all)]
4344async fn run_http_server(
4345    addr: SocketAddr,
4346    storage: Arc<Storage>,
4347    auth: Arc<AuthHandle>,
4348    audit: Arc<Audit>,
4349    metadata: Arc<RuntimeMetadata>,
4350    cors: Option<CorsConfig>,
4351) -> Result<()> {
4352    let listener = TcpListener::bind(addr)
4353        .await
4354        .with_context(|| format!("failed to bind HTTP listener on {addr}"))?;
4355
4356    // Allow explicit opt-in for local UI origins; default is no cross-origin access.
4357    let cors_layer = cors.as_ref().map(|cfg| cfg.to_layer());
4358    let router = http::router(storage, auth, audit, metadata);
4359    let app = match cors_layer {
4360        Some(layer) => router.layer(layer),
4361        None => router,
4362    };
4363
4364    info!(%addr, "starting runtime HTTP API server");
4365
4366    let std_listener = listener
4367        .into_std()
4368        .context("failed to convert HTTP listener for hyper server")?;
4369    HyperServer::from_tcp(std_listener)
4370        .context("failed to build runtime HTTP server")?
4371        .serve(app.into_make_service())
4372        .await
4373        .context("runtime HTTP API server exited unexpectedly")
4374}
4375
4376#[derive(Clone, Debug)]
4377struct CorsConfig {
4378    origins: Vec<HeaderValue>,
4379    display: Vec<String>,
4380}
4381
4382impl CorsConfig {
4383    fn to_layer(&self) -> CorsLayer {
4384        CorsLayer::new()
4385            .allow_origin(self.origins.clone())
4386            .allow_methods([Method::GET, Method::POST, Method::OPTIONS])
4387            .allow_headers([header::CONTENT_TYPE, header::AUTHORIZATION])
4388    }
4389}
4390
4391fn cors_config_from_env() -> Option<CorsConfig> {
4392    let raw = std::env::var("FLEETFORGE_CORS_ALLOW_ORIGINS").ok()?;
4393    let mut origins = Vec::new();
4394    let mut display = Vec::new();
4395    for origin in raw.split(',') {
4396        let trimmed = origin.trim();
4397        if trimmed.is_empty() {
4398            continue;
4399        }
4400        match HeaderValue::from_str(trimmed) {
4401            Ok(value) => {
4402                origins.push(value);
4403                display.push(trimmed.to_string());
4404            }
4405            Err(err) => {
4406                warn!(origin = %trimmed, %err, "invalid origin in FLEETFORGE_CORS_ALLOW_ORIGINS")
4407            }
4408        }
4409    }
4410
4411    if origins.is_empty() {
4412        warn!(
4413            "no valid origins parsed from FLEETFORGE_CORS_ALLOW_ORIGINS; CORS will remain disabled"
4414        );
4415        return None;
4416    }
4417
4418    Some(CorsConfig { origins, display })
4419}
4420
4421fn transparency_writer_enabled_flag() -> bool {
4422    let enabled = matches!(
4423        std::env::var(TRANSPARENCY_WRITER_FLAG)
4424            .ok()
4425            .map(|value| value.trim().to_ascii_lowercase())
4426            .unwrap_or_default()
4427            .as_str(),
4428        "1" | "true" | "yes" | "on"
4429    );
4430    if enabled && !feature_allowed(LicensedFeature::ScittTransparency) {
4431        warn!(
4432            "FLEETFORGE_TRANSPARENCY_WRITER=1 but the current license tier does not include SCITT publishing; disabling writer"
4433        );
4434        return false;
4435    }
4436    enabled
4437}
4438
4439fn transparency_scope_from_env() -> TransparencyScope {
4440    match std::env::var("FLEETFORGE_TRANSPARENCY_SCOPE")
4441        .ok()
4442        .map(|value| value.trim().to_ascii_lowercase())
4443        .as_deref()
4444    {
4445        Some("artifacts") => TransparencyScope::Artifacts,
4446        Some("runs") => TransparencyScope::Runs,
4447        Some("gates") => TransparencyScope::Gates,
4448        Some("disabled") => TransparencyScope::Disabled,
4449        _ => TransparencyScope::Gates,
4450    }
4451}
4452
4453fn transparency_writer_interval() -> Duration {
4454    let interval = std::env::var(TRANSPARENCY_WRITER_INTERVAL_ENV)
4455        .ok()
4456        .and_then(|value| value.parse::<u64>().ok())
4457        .unwrap_or(TRANSPARENCY_WRITER_DEFAULT_INTERVAL_SECS);
4458    Duration::from_secs(interval.max(TRANSPARENCY_WRITER_INTERVAL_MIN_SECS))
4459}
4460
4461fn parse_dag_json(input: &str) -> Result<Value, Status> {
4462    serde_json::from_str(input)
4463        .map_err(|err| invalid_argument(format!("invalid dag_json payload: {err}")))
4464}
4465
4466fn extract_priority(step_obj: &Map<String, Value>) -> i16 {
4467    step_obj
4468        .get("policy")
4469        .and_then(Value::as_object)
4470        .and_then(|policy| policy.get("priority"))
4471        .and_then(Value::as_i64)
4472        .unwrap_or(0)
4473        .clamp(i16::MIN as i64, i16::MAX as i64) as i16
4474}
4475
4476fn build_run_detail(
4477    run: models::Run,
4478    steps: Vec<models::Step>,
4479) -> Result<runtime::RunDetail, Status> {
4480    let (inputs_value, labels_map) = unpack_input_ctx(&run.input_ctx);
4481    let breakpoint_specs = extract_breakpoint_specs(&run.input_ctx);
4482    let inputs_struct = match &inputs_value {
4483        Value::Null => None,
4484        Value::Object(map) if map.is_empty() => None,
4485        Value::Object(_) => Some(json_to_prost_struct(&inputs_value).map_err(|err| {
4486            internal_error(format!("failed to encode inputs for response: {err}"))
4487        })?),
4488        _ => None,
4489    };
4490
4491    let dag_json = serde_json::to_string(&run.dag_json).map_err(|err| {
4492        internal_error(format!("failed to serialise dag_json for response: {err}"))
4493    })?;
4494
4495    let run_spec = runtime::RunSpec {
4496        dag_json,
4497        inputs: inputs_struct,
4498        seed: run.seed,
4499        labels: labels_map,
4500        breakpoints: breakpoint_specs,
4501    };
4502
4503    let step_summaries = steps
4504        .into_iter()
4505        .map(step_to_summary)
4506        .collect::<Result<Vec<_>, Status>>()?;
4507
4508    Ok(runtime::RunDetail {
4509        run_id: run.run_id.to_string(),
4510        status: map_run_status(run.status) as i32,
4511        spec: Some(run_spec),
4512        created_at: Some(to_timestamp(run.created_at)),
4513        updated_at: None,
4514        steps: step_summaries,
4515    })
4516}
4517
4518fn unpack_input_ctx(ctx: &Value) -> (Value, HashMap<String, String>) {
4519    match ctx {
4520        Value::Object(map) => {
4521            let inputs = map
4522                .get("inputs")
4523                .cloned()
4524                .unwrap_or_else(|| Value::Object(Map::new()));
4525            let labels = map
4526                .get("labels")
4527                .map(labels_value_to_map)
4528                .unwrap_or_default();
4529            (inputs, labels)
4530        }
4531        other => (other.clone(), HashMap::new()),
4532    }
4533}
4534
4535fn extract_breakpoint_specs(ctx: &Value) -> Vec<String> {
4536    let Value::Object(map) = ctx else {
4537        return Vec::new();
4538    };
4539    let Some(Value::Object(debug)) = map.get("debug") else {
4540        return Vec::new();
4541    };
4542
4543    if let Some(Value::Array(specs)) = debug.get("breakpoint_specs") {
4544        return specs
4545            .iter()
4546            .filter_map(|value| value.as_str().map(|s| s.to_string()))
4547            .collect();
4548    }
4549
4550    if let Some(Value::Array(entries)) = debug.get("breakpoints") {
4551        return entries
4552            .iter()
4553            .filter_map(|entry| {
4554                entry
4555                    .get("spec")
4556                    .and_then(Value::as_str)
4557                    .map(|s| s.to_string())
4558            })
4559            .collect();
4560    }
4561
4562    Vec::new()
4563}
4564
4565fn load_debug_breakpoints(debug: &Map<String, Value>) -> Vec<(String, Uuid)> {
4566    debug
4567        .get("breakpoints")
4568        .and_then(Value::as_array)
4569        .map(|items| {
4570            items
4571                .iter()
4572                .filter_map(|entry| {
4573                    let obj = entry.as_object()?;
4574                    let spec = obj.get("spec")?.as_str()?.to_string();
4575                    let step_id_str = obj.get("step_id")?.as_str()?;
4576                    let step_id = Uuid::parse_str(step_id_str).ok()?;
4577                    Some((spec, step_id))
4578                })
4579                .collect()
4580        })
4581        .unwrap_or_default()
4582}
4583
4584fn store_debug_breakpoints(debug: &mut Map<String, Value>, entries: &[(String, Uuid)]) {
4585    let list = entries
4586        .iter()
4587        .map(|(spec, step_id)| {
4588            let mut obj = Map::new();
4589            obj.insert("spec".to_string(), Value::String(spec.clone()));
4590            obj.insert("step_id".to_string(), Value::String(step_id.to_string()));
4591            Value::Object(obj)
4592        })
4593        .collect::<Vec<_>>();
4594    debug.insert("breakpoints".to_string(), Value::Array(list));
4595
4596    let specs = entries
4597        .iter()
4598        .map(|(spec, _)| Value::String(spec.clone()))
4599        .collect::<Vec<_>>();
4600    debug.insert("breakpoint_specs".to_string(), Value::Array(specs));
4601}
4602
4603fn load_slug_map(debug: &Map<String, Value>) -> HashMap<String, Uuid> {
4604    debug
4605        .get("slugs")
4606        .and_then(Value::as_object)
4607        .map(|map| {
4608            map.iter()
4609                .filter_map(|(slug, value)| {
4610                    value
4611                        .as_str()
4612                        .and_then(|id| Uuid::parse_str(id).ok())
4613                        .map(|uuid| (slug.clone(), uuid))
4614                })
4615                .collect()
4616        })
4617        .unwrap_or_default()
4618}
4619
4620fn store_slug_map(debug: &mut Map<String, Value>, entries: &[(String, Uuid)]) {
4621    let mut slugs = Map::new();
4622    for (slug, step_id) in entries {
4623        slugs.insert(slug.clone(), Value::String(step_id.to_string()));
4624    }
4625    debug.insert("slugs".to_string(), Value::Object(slugs));
4626}
4627
4628fn extract_active_pause_step_id(debug: &Map<String, Value>) -> Option<Uuid> {
4629    debug
4630        .get("active_pause")
4631        .and_then(|value| value.as_object())
4632        .and_then(|obj| obj.get("step_id"))
4633        .and_then(Value::as_str)
4634        .and_then(|s| Uuid::parse_str(s).ok())
4635}
4636
4637fn merge_json_object(target: &mut Map<String, Value>, patch: &Map<String, Value>) {
4638    for (key, value) in patch {
4639        target.insert(key.clone(), value.clone());
4640    }
4641}
4642
4643fn collect_step_ids(run: &models::Run) -> HashSet<Uuid> {
4644    run.dag_json
4645        .get("steps")
4646        .and_then(Value::as_array)
4647        .map(|steps| {
4648            steps
4649                .iter()
4650                .filter_map(|step| {
4651                    step.get("id")
4652                        .and_then(Value::as_str)
4653                        .and_then(|s| Uuid::parse_str(s).ok())
4654                })
4655                .collect()
4656        })
4657        .unwrap_or_default()
4658}
4659
4660fn collect_slug_entries(run: &models::Run) -> Vec<(String, Uuid)> {
4661    run.dag_json
4662        .get("steps")
4663        .and_then(Value::as_array)
4664        .map(|steps| {
4665            steps
4666                .iter()
4667                .filter_map(|step| {
4668                    let id = step
4669                        .get("id")
4670                        .and_then(Value::as_str)
4671                        .and_then(|s| Uuid::parse_str(s).ok())?;
4672                    let slug = step
4673                        .get("slug")
4674                        .and_then(Value::as_str)
4675                        .map(|s| s.trim().to_string())
4676                        .filter(|s| !s.is_empty())?;
4677                    Some((slug, id))
4678                })
4679                .collect()
4680        })
4681        .unwrap_or_default()
4682}
4683
4684fn build_branch_dag(dag_json: &Value, start: Uuid) -> Result<Value, SubmitRunError> {
4685    let dag_map = dag_json
4686        .as_object()
4687        .cloned()
4688        .ok_or_else(|| SubmitRunError::invalid("dag_json must be a JSON object"))?;
4689    let steps_array = dag_map
4690        .get("steps")
4691        .and_then(Value::as_array)
4692        .ok_or_else(|| SubmitRunError::invalid("dag_json.steps must be an array"))?;
4693    let edges_array = dag_map
4694        .get("edges")
4695        .and_then(Value::as_array)
4696        .ok_or_else(|| SubmitRunError::invalid("dag_json.edges must be an array"))?;
4697
4698    let mut step_defs: HashMap<Uuid, Value> = HashMap::new();
4699    for step in steps_array {
4700        let step_obj = step
4701            .as_object()
4702            .ok_or_else(|| SubmitRunError::invalid("each step must be an object"))?;
4703        let id = step_obj
4704            .get("id")
4705            .and_then(Value::as_str)
4706            .ok_or_else(|| SubmitRunError::invalid("step.id missing"))?;
4707        let step_id = Uuid::parse_str(id)
4708            .map_err(|err| SubmitRunError::invalid(format!("invalid step.id: {err}")))?;
4709        step_defs.insert(step_id, step.clone());
4710    }
4711
4712    if !step_defs.contains_key(&start) {
4713        return Err(SubmitRunError::invalid("from_step references unknown step"));
4714    }
4715
4716    let mut adjacency: HashMap<Uuid, Vec<Uuid>> = HashMap::new();
4717    for edge in edges_array {
4718        let pair = edge
4719            .as_array()
4720            .ok_or_else(|| SubmitRunError::invalid("edges entries must be arrays"))?;
4721        if pair.len() != 2 {
4722            return Err(SubmitRunError::invalid(
4723                "edges entries must contain two members",
4724            ));
4725        }
4726        let from = pair[0]
4727            .as_str()
4728            .ok_or_else(|| SubmitRunError::invalid("edge source must be a string"))?;
4729        let to = pair[1]
4730            .as_str()
4731            .ok_or_else(|| SubmitRunError::invalid("edge target must be a string"))?;
4732        let from_id = Uuid::parse_str(from)
4733            .map_err(|err| SubmitRunError::invalid(format!("invalid edge source: {err}")))?;
4734        let to_id = Uuid::parse_str(to)
4735            .map_err(|err| SubmitRunError::invalid(format!("invalid edge target: {err}")))?;
4736        adjacency.entry(from_id).or_default().push(to_id);
4737    }
4738
4739    let mut queue = VecDeque::new();
4740    let mut visited: HashSet<Uuid> = HashSet::new();
4741    queue.push_back(start);
4742    while let Some(node) = queue.pop_front() {
4743        if !visited.insert(node) {
4744            continue;
4745        }
4746        if let Some(neighbors) = adjacency.get(&node) {
4747            for next in neighbors {
4748                queue.push_back(*next);
4749            }
4750        }
4751    }
4752
4753    let filtered_steps = steps_array
4754        .iter()
4755        .filter_map(|step| {
4756            let id = step
4757                .get("id")
4758                .and_then(Value::as_str)
4759                .and_then(|s| Uuid::parse_str(s).ok());
4760            match id {
4761                Some(step_id) if visited.contains(&step_id) => Some(step.clone()),
4762                _ => None,
4763            }
4764        })
4765        .collect::<Vec<_>>();
4766
4767    let filtered_edges = edges_array
4768        .iter()
4769        .filter(|edge| {
4770            if let Some(pair) = edge.as_array() {
4771                if pair.len() == 2 {
4772                    if let (Some(from), Some(to)) = (
4773                        pair[0].as_str().and_then(|s| Uuid::parse_str(s).ok()),
4774                        pair[1].as_str().and_then(|s| Uuid::parse_str(s).ok()),
4775                    ) {
4776                        return visited.contains(&from) && visited.contains(&to);
4777                    }
4778                }
4779            }
4780            false
4781        })
4782        .cloned()
4783        .collect::<Vec<_>>();
4784
4785    let mut new_dag = dag_json.as_object().cloned().unwrap_or_default();
4786    new_dag.insert("steps".to_string(), Value::Array(filtered_steps));
4787    new_dag.insert("edges".to_string(), Value::Array(filtered_edges));
4788    Ok(Value::Object(new_dag))
4789}
4790
4791fn collect_prompt_packs_json(dag_json: &Value) -> Vec<String> {
4792    match dag_json {
4793        Value::Object(map) => {
4794            let packs = map.get("prompt_packs").or_else(|| map.get("promptPacks"));
4795            match packs {
4796                Some(Value::Array(items)) => items
4797                    .iter()
4798                    .filter_map(|item| item.as_str().map(|s| s.to_string()))
4799                    .collect(),
4800                Some(Value::String(value)) => vec![value.clone()],
4801                _ => Vec::new(),
4802            }
4803        }
4804        _ => Vec::new(),
4805    }
4806}
4807
4808struct DriftComputation {
4809    token_delta: Option<f64>,
4810    cost_delta: Option<f64>,
4811    within_tolerance: bool,
4812}
4813
4814const DEFAULT_TOKEN_TOLERANCE: f64 = 0.0;
4815const DEFAULT_COST_TOLERANCE: f64 = 0.0;
4816const TOKEN_KEYS: &[&str] = &["tokens", "token_count", "total_tokens"];
4817const COST_KEYS: &[&str] = &["cost", "cost_usd", "total_cost"];
4818const SANITIZE_KEYS: &[&str] = &[
4819    "tokens",
4820    "token_count",
4821    "total_tokens",
4822    "cost",
4823    "cost_usd",
4824    "total_cost",
4825];
4826
4827fn wrap_attempt_output(attempt: Option<&models::StepAttempt>, output: &Value) -> Value {
4828    match attempt {
4829        Some(record) => json!({
4830            "attempt": record.attempt,
4831            "status": record.status.as_str(),
4832            "recorded_at": record.recorded_at,
4833            "output": output,
4834        }),
4835        None => output.clone(),
4836    }
4837}
4838
4839fn compute_drift(
4840    recorded: &Value,
4841    current: &Value,
4842    token_tolerance: f64,
4843    cost_tolerance: f64,
4844) -> DriftComputation {
4845    let token_delta = diff_numeric(recorded, current, TOKEN_KEYS);
4846    let cost_delta = diff_numeric(recorded, current, COST_KEYS);
4847
4848    let sanitized_recorded = strip_keys(recorded, SANITIZE_KEYS);
4849    let sanitized_current = strip_keys(current, SANITIZE_KEYS);
4850
4851    let tokens_within = token_delta
4852        .map(|delta| delta.abs() <= token_tolerance)
4853        .unwrap_or(true);
4854    let cost_within = cost_delta
4855        .map(|delta| delta.abs() <= cost_tolerance)
4856        .unwrap_or(true);
4857    let structure_equal = sanitized_recorded == sanitized_current;
4858
4859    DriftComputation {
4860        token_delta,
4861        cost_delta,
4862        within_tolerance: tokens_within && cost_within && structure_equal,
4863    }
4864}
4865
4866fn diff_numeric(recorded: &Value, current: &Value, keys: &[&str]) -> Option<f64> {
4867    let recorded_value = find_numeric(recorded, keys)?;
4868    let current_value = find_numeric(current, keys)?;
4869    Some(current_value - recorded_value)
4870}
4871
4872fn find_numeric(value: &Value, keys: &[&str]) -> Option<f64> {
4873    match value {
4874        Value::Object(map) => {
4875            for key in keys {
4876                if let Some(v) = map.get(*key) {
4877                    if let Some(num) = v.as_f64() {
4878                        return Some(num);
4879                    }
4880                }
4881            }
4882            for v in map.values() {
4883                if let Some(found) = find_numeric(v, keys) {
4884                    return Some(found);
4885                }
4886            }
4887            None
4888        }
4889        Value::Array(items) => {
4890            for item in items {
4891                if let Some(found) = find_numeric(item, keys) {
4892                    return Some(found);
4893                }
4894            }
4895            None
4896        }
4897        _ => None,
4898    }
4899}
4900
4901fn strip_keys(value: &Value, keys: &[&str]) -> Value {
4902    match value {
4903        Value::Object(map) => {
4904            let mut new_map = serde_json::Map::new();
4905            for (k, v) in map {
4906                if keys.iter().any(|key| key == k) {
4907                    continue;
4908                }
4909                new_map.insert(k.clone(), strip_keys(v, keys));
4910            }
4911            Value::Object(new_map)
4912        }
4913        Value::Array(items) => Value::Array(items.iter().map(|v| strip_keys(v, keys)).collect()),
4914        _ => value.clone(),
4915    }
4916}
4917
4918fn step_to_summary(step: models::Step) -> Result<runtime::StepSummary, Status> {
4919    let step_type = step
4920        .spec_json
4921        .get("type")
4922        .and_then(Value::as_str)
4923        .unwrap_or_default()
4924        .to_string();
4925
4926    let output = match step.output_json {
4927        Some(value) => Some(json_to_prost_value(&value).map_err(|err| {
4928            internal_error(format!(
4929                "invalid output payload for step {}: {err}",
4930                step.step_id
4931            ))
4932        })?),
4933        None => None,
4934    };
4935
4936    let error = match step.error_json {
4937        Some(value) => Some(json_to_prost_value(&value).map_err(|err| {
4938            internal_error(format!(
4939                "invalid error payload for step {}: {err}",
4940                step.step_id
4941            ))
4942        })?),
4943        None => None,
4944    };
4945
4946    Ok(runtime::StepSummary {
4947        step_id: step.step_id.to_string(),
4948        idx: step.idx,
4949        r#type: step_type,
4950        status: step.status.as_str().to_string(),
4951        attempt: step.attempt,
4952        leased_at: None,
4953        completed_at: None,
4954        output,
4955        error,
4956    })
4957}
4958
4959fn run_event_from_outbox(event: models::OutboxEvent) -> Result<runtime::RunEvent, Status> {
4960    let payload = json_to_prost_value(&event.payload).map_err(|err| {
4961        internal_error(format!(
4962            "failed to encode outbox payload for event {}: {err}",
4963            event.id
4964        ))
4965    })?;
4966
4967    Ok(runtime::RunEvent {
4968        run_id: event.run_id.to_string(),
4969        step_id: event.step_id.map(|id| id.to_string()).unwrap_or_default(),
4970        kind: event.kind,
4971        payload: Some(payload),
4972        occurred_at: Some(to_timestamp(event.created_at)),
4973    })
4974}
4975
4976fn map_run_status(status: models::RunStatus) -> runtime::RunStatus {
4977    match status {
4978        models::RunStatus::Pending => runtime::RunStatus::Pending,
4979        models::RunStatus::Running => runtime::RunStatus::Running,
4980        models::RunStatus::Succeeded => runtime::RunStatus::Succeeded,
4981        models::RunStatus::Failed => runtime::RunStatus::Failed,
4982        models::RunStatus::Canceled => runtime::RunStatus::Canceled,
4983        models::RunStatus::PausedAtStep => runtime::RunStatus::PausedAtStep,
4984    }
4985}
4986
4987#[derive(Clone, Debug)]
4988pub(crate) struct EvalScenarioSpecData {
4989    pack: String,
4990    slug: String,
4991    title: Option<String>,
4992    description: Option<String>,
4993    run_spec: Value,
4994    expected: Value,
4995}
4996
4997pub(crate) fn parse_eval_spec(spec: &Value) -> anyhow::Result<EvalScenarioSpecData> {
4998    let pack = spec
4999        .get("pack")
5000        .and_then(Value::as_str)
5001        .ok_or_else(|| anyhow!("scenario spec missing 'pack' field"))?
5002        .to_string();
5003    let slug = spec
5004        .get("slug")
5005        .and_then(Value::as_str)
5006        .ok_or_else(|| anyhow!("scenario spec missing 'slug' field"))?
5007        .to_string();
5008    let title = spec
5009        .get("title")
5010        .and_then(Value::as_str)
5011        .map(|value| value.to_string());
5012    let description = spec
5013        .get("description")
5014        .and_then(Value::as_str)
5015        .map(|value| value.to_string());
5016    let run_spec = spec
5017        .get("run_spec")
5018        .cloned()
5019        .ok_or_else(|| anyhow!("scenario spec missing 'run_spec' payload"))?;
5020    if !run_spec.is_object() {
5021        return Err(anyhow!("scenario spec 'run_spec' must be a JSON object"));
5022    }
5023    let expected = spec
5024        .get("expected")
5025        .cloned()
5026        .ok_or_else(|| anyhow!("scenario spec missing 'expected' payload"))?;
5027    if !expected.is_object() {
5028        return Err(anyhow!("scenario spec 'expected' must be a JSON object"));
5029    }
5030
5031    Ok(EvalScenarioSpecData {
5032        pack,
5033        slug,
5034        title,
5035        description,
5036        run_spec,
5037        expected,
5038    })
5039}
5040
5041fn metric_summary(current: f64, previous: Option<f64>) -> Value {
5042    let delta = previous.map(|p| current - p);
5043    json!({
5044        "value": current,
5045        "previous": previous,
5046        "delta": delta,
5047    })
5048}
5049
5050fn extract_metric(metrics: &Value, key: &str) -> Option<f64> {
5051    metrics
5052        .as_object()
5053        .and_then(|map| map.get(key))
5054        .and_then(|entry| entry.get("value"))
5055        .and_then(Value::as_f64)
5056}
5057
5058fn compute_eval_metrics(
5059    spec: &EvalScenarioSpecData,
5060    run: &models::Run,
5061    steps: &[models::Step],
5062    last_event_time: Option<DateTime<Utc>>,
5063    previous_metrics: Option<&Value>,
5064    commit: Option<String>,
5065) -> anyhow::Result<(String, Value)> {
5066    let tokens_actual: f64 = steps
5067        .iter()
5068        .map(|step| {
5069            step.actual_tokens
5070                .map(|value| value as f64)
5071                .unwrap_or_else(|| step.estimated_tokens as f64)
5072        })
5073        .sum();
5074    let cost_actual: f64 = steps
5075        .iter()
5076        .map(|step| step.actual_cost.unwrap_or(step.estimated_cost))
5077        .sum();
5078
5079    let run_end = last_event_time.unwrap_or(run.created_at);
5080    let latency_ms = run_end
5081        .signed_duration_since(run.created_at)
5082        .num_milliseconds()
5083        .max(0) as f64;
5084
5085    let previous_tokens = previous_metrics.and_then(|metrics| extract_metric(metrics, "tokens"));
5086    let previous_cost = previous_metrics.and_then(|metrics| extract_metric(metrics, "cost_usd"));
5087    let previous_latency =
5088        previous_metrics.and_then(|metrics| extract_metric(metrics, "latency_ms"));
5089
5090    let mut differences: Vec<Value> = Vec::new();
5091    if let Some(expected_status) = spec.expected.get("status").and_then(Value::as_str) {
5092        let actual_status = run.status.as_str();
5093        if !expected_status.eq_ignore_ascii_case(actual_status) {
5094            differences.push(json!({
5095                "type": "status",
5096                "expected": expected_status,
5097                "actual": actual_status,
5098            }));
5099        }
5100    }
5101
5102    let mut actual_steps: HashMap<String, Value> = HashMap::with_capacity(steps.len());
5103    for step in steps {
5104        if let Some(output) = step.output_json.clone() {
5105            actual_steps.insert(step.step_id.to_string(), output);
5106        }
5107    }
5108
5109    if let Some(expected_steps) = spec.expected.get("steps").and_then(Value::as_object) {
5110        for (step_id, expected_output) in expected_steps {
5111            match actual_steps.get(step_id) {
5112                Some(actual_output) if actual_output == expected_output => {}
5113                Some(actual_output) => {
5114                    differences.push(json!({
5115                        "type": "step",
5116                        "step_id": step_id,
5117                        "expected": expected_output,
5118                        "actual": actual_output,
5119                    }));
5120                }
5121                None => {
5122                    differences.push(json!({
5123                        "type": "step_missing",
5124                        "step_id": step_id,
5125                        "expected": expected_output,
5126                    }));
5127                }
5128            }
5129        }
5130    }
5131
5132    let quality_matched = differences.is_empty();
5133    let outcome = if quality_matched { "passed" } else { "failed" }.to_string();
5134
5135    let mut metrics_map = Map::new();
5136    if let Some(commit) = commit {
5137        if !commit.is_empty() {
5138            metrics_map.insert("commit".to_string(), Value::String(commit));
5139        }
5140    }
5141    metrics_map.insert(
5142        "tokens".to_string(),
5143        metric_summary(tokens_actual, previous_tokens),
5144    );
5145    metrics_map.insert(
5146        "cost_usd".to_string(),
5147        metric_summary(cost_actual, previous_cost),
5148    );
5149    metrics_map.insert(
5150        "latency_ms".to_string(),
5151        metric_summary(latency_ms, previous_latency),
5152    );
5153    metrics_map.insert(
5154        "quality".to_string(),
5155        Value::Object({
5156            let mut quality = Map::new();
5157            quality.insert("matched".to_string(), Value::Bool(quality_matched));
5158            quality.insert("differences".to_string(), Value::Array(differences));
5159            quality
5160        }),
5161    );
5162
5163    Ok((outcome, Value::Object(metrics_map)))
5164}
5165
5166fn build_eval_result_proto(result: models::EvalResult) -> Result<runtime::EvalResult, Status> {
5167    let metrics_struct = json_to_prost_struct(&result.metrics)
5168        .map_err(|err| internal_error(format!("invalid eval metrics payload: {err}")))?;
5169    let commit = result
5170        .metrics
5171        .get("commit")
5172        .and_then(Value::as_str)
5173        .unwrap_or("")
5174        .to_string();
5175
5176    Ok(runtime::EvalResult {
5177        result_id: result.result_id.to_string(),
5178        run_id: result.run_id.to_string(),
5179        scenario_id: result.scenario_id.to_string(),
5180        outcome: result.outcome,
5181        commit,
5182        executed_at: Some(to_timestamp(result.executed_at)),
5183        metrics: Some(metrics_struct),
5184    })
5185}
5186
5187fn build_eval_scenario_proto(
5188    scenario: &models::EvalScenario,
5189    spec: &EvalScenarioSpecData,
5190    history: Vec<runtime::EvalResult>,
5191) -> Result<runtime::EvalScenario, Status> {
5192    Ok(runtime::EvalScenario {
5193        scenario_id: scenario.scenario_id.to_string(),
5194        pack: spec.pack.clone(),
5195        slug: spec.slug.clone(),
5196        title: spec.title.clone().unwrap_or_else(|| spec.slug.clone()),
5197        description: spec.description.clone().unwrap_or_default(),
5198        created_at: Some(to_timestamp(scenario.created_at)),
5199        history,
5200    })
5201}
5202
5203#[cfg(test)]
5204mod unit_tests {
5205    use super::*;
5206    use chrono::Utc;
5207    use fleetforge_common::prost_json::prost_value_to_json;
5208    use fleetforge_storage::models;
5209    use serde_json::{json, Map, Value};
5210    use std::collections::{HashMap, VecDeque};
5211    use std::time::Duration;
5212    use tonic::Code;
5213    use uuid::Uuid;
5214
5215    #[test]
5216    fn parse_page_token_supports_defaults() {
5217        assert_eq!(parse_page_token("").expect("empty token"), 0);
5218        assert_eq!(parse_page_token("0").expect("zero token"), 0);
5219        assert!(parse_page_token("abc").is_err());
5220    }
5221
5222    #[test]
5223    fn query_filters_normalizes_inputs() {
5224        let run_id = Uuid::new_v4();
5225        let mut request = QueryEventsRequest::default();
5226        request.run_ids.push(run_id.to_string());
5227        request.kinds.push("Step_Succeeded".to_string());
5228        request.workspace_ids.push("Workspace-1".to_string());
5229        request
5230            .policy_effects
5231            .push(PolicyDecisionEffect::PolicyDecisionEffectDeny as i32);
5232        request.severities.push(EventSeverity::Error as i32);
5233
5234        let filters = QueryFiltersOwned::from_request(&request).expect("filters");
5235        assert_eq!(filters.run_ids, vec![run_id]);
5236        assert_eq!(filters.kinds, vec!["step_succeeded"]);
5237        assert_eq!(filters.workspace_ids, vec!["workspace-1"]);
5238        assert_eq!(
5239            filters.policy_effects,
5240            vec![PolicyDecisionEffect::PolicyDecisionEffectDeny]
5241        );
5242        assert_eq!(filters.severities, vec![EventSeverity::Error]);
5243    }
5244
5245    #[test]
5246    fn matches_filters_respects_workspace_and_kind() {
5247        let run_id = Uuid::new_v4();
5248        let mut request = QueryEventsRequest::default();
5249        request.run_ids.push(run_id.to_string());
5250        request.workspace_ids.push("tenant-x".to_string());
5251        request.kinds.push("run_succeeded".to_string());
5252        let filters = QueryFiltersOwned::from_request(&request).expect("filters");
5253
5254        let payload = json!({"trace": {"run": {"workspace_id": "Tenant-X"}}});
5255        let kind = "run_succeeded";
5256        let severity = EventSeverity::Info;
5257        assert!(
5258            matches_filters(&filters, &payload, kind, severity, None),
5259            "matching workspace should pass filter"
5260        );
5261
5262        let other_payload = json!({"trace": {"run": {"workspace_id": "tenant-y"}}});
5263        assert!(
5264            !matches_filters(&filters, &other_payload, kind, severity, None),
5265            "non-matching workspace should fail filter"
5266        );
5267    }
5268
5269    #[test]
5270    fn parse_dag_json_round_trip() {
5271        let dag = r#"{"steps": []}"#;
5272        let parsed = parse_dag_json(dag).expect("valid JSON");
5273        assert_eq!(parsed["steps"], Value::Array(vec![]));
5274
5275        let err = parse_dag_json("not valid json").expect_err("should reject invalid JSON");
5276        assert_eq!(err.code(), Code::InvalidArgument);
5277    }
5278
5279    #[test]
5280    fn extract_priority_clamps_to_i16() {
5281        let mut policy = Map::new();
5282        policy.insert("priority".to_string(), Value::from(40_000));
5283        let mut step = Map::new();
5284        step.insert("policy".to_string(), Value::Object(policy));
5285        let priority = extract_priority(&step);
5286        assert_eq!(priority, i16::MAX);
5287    }
5288
5289    #[test]
5290    fn extract_priority_defaults_to_zero() {
5291        let step = Map::new();
5292        let priority = extract_priority(&step);
5293        assert_eq!(priority, 0);
5294    }
5295
5296    #[test]
5297    fn extract_priority_handles_negative_overflow() {
5298        let mut policy = Map::new();
5299        policy.insert("priority".to_string(), Value::from(-50_000));
5300        let mut step = Map::new();
5301        step.insert("policy".to_string(), Value::Object(policy));
5302        let priority = extract_priority(&step);
5303        assert_eq!(priority, i16::MIN);
5304    }
5305
5306    #[test]
5307    fn labels_map_round_trip() {
5308        let mut map = HashMap::new();
5309        map.insert("team".to_string(), "infra".to_string());
5310        map.insert("env".to_string(), "dev".to_string());
5311        let value = labels_map_to_value(&map);
5312        let round_trip = labels_value_to_map(&value);
5313        assert_eq!(map, round_trip);
5314    }
5315
5316    #[test]
5317    fn build_run_detail_converts_payloads() {
5318        let run_id = Uuid::new_v4();
5319        let step_id = Uuid::new_v4();
5320        let now = Utc::now();
5321
5322        let run = models::Run {
5323            run_id,
5324            created_at: now,
5325            status: models::RunStatus::Pending,
5326            dag_json: json!({"steps": []}),
5327            input_ctx: json!({
5328                "inputs": {"foo": "bar"},
5329                "labels": {"team": "infra"}
5330            }),
5331            seed: 7,
5332            idempotency_key: Some("abc".into()),
5333        };
5334
5335        let step = models::Step {
5336            run_id,
5337            step_id,
5338            idx: 0,
5339            priority: 0,
5340            spec_json: json!({"id": step_id, "type": "tool", "inputs": {}, "policy": {}}),
5341            status: models::StepStatus::Succeeded,
5342            attempt: 1,
5343            created_at: now,
5344            pending_dependencies: 0,
5345            total_dependencies: 0,
5346            estimated_tokens: 0,
5347            estimated_cost: 0.0,
5348            actual_tokens: Some(10),
5349            actual_cost: Some(0.01),
5350            leased_by: None,
5351            lease_expires_at: None,
5352            output_json: Some(json!({"answer": 42})),
5353            error_json: Some(json!({"message": "none"})),
5354            input_snapshot: None,
5355            output_snapshot: None,
5356            provider: Some("test".into()),
5357            provider_version: Some("1.0".into()),
5358        };
5359
5360        let detail = build_run_detail(run, vec![step]).expect("build run detail");
5361        let spec = detail.spec.expect("spec present");
5362        assert_eq!(spec.dag_json, r#"{"steps":[]}"#);
5363        assert_eq!(spec.seed, 7);
5364        assert_eq!(spec.labels.get("team"), Some(&"infra".to_string()));
5365
5366        let step_summary = &detail.steps[0];
5367        assert_eq!(step_summary.step_id, step_id.to_string());
5368        assert_eq!(step_summary.status, "succeeded");
5369        let output_value = prost_value_to_json(step_summary.output.as_ref().unwrap());
5370        assert_eq!(output_value, json!({"answer": 42}));
5371    }
5372
5373    #[test]
5374    fn subscription_config_parses_filters() {
5375        let run_id = Uuid::new_v4().to_string();
5376        let step_id = Uuid::new_v4().to_string();
5377        let open = SubscribeRunOpen {
5378            run_ids: vec![run_id.clone()],
5379            step_ids: vec![step_id.clone()],
5380            kinds: vec!["step_succeeded".to_string()],
5381            severities: vec![EventSeverity::Warn as i32],
5382            tags: vec!["team:core".to_string()],
5383            since_offset: 10,
5384            batch_size: 32,
5385            max_inflight: 20,
5386            heartbeat_interval_ms: 5_000,
5387        };
5388
5389        let config = SubscriptionConfig::try_from_open(open).expect("valid config");
5390        assert_eq!(config.run_id.to_string(), run_id);
5391        assert!(config
5392            .step_ids
5393            .as_ref()
5394            .expect("step filter")
5395            .contains(&Uuid::parse_str(&step_id).unwrap()));
5396        assert!(config
5397            .kinds
5398            .as_ref()
5399            .expect("kind filter")
5400            .contains("step_succeeded"));
5401        assert!(config
5402            .severities
5403            .as_ref()
5404            .expect("severity filter")
5405            .contains(&EventSeverity::Warn));
5406        assert!(config
5407            .tags
5408            .as_ref()
5409            .expect("tag filter")
5410            .contains("team:core"));
5411        assert_eq!(config.since_offset, 10);
5412        assert_eq!(config.batch_size, 32);
5413        assert_eq!(config.max_inflight, 20);
5414        assert_eq!(config.heartbeat.as_millis(), 5_000);
5415    }
5416
5417    #[test]
5418    fn handle_ack_evicts_inflight_offsets() {
5419        let mut inflight = VecDeque::from(vec![5, 9, 12]);
5420        let mut last_ack = 0;
5421        handle_ack(&mut inflight, &mut last_ack, 9);
5422        assert_eq!(last_ack, 9);
5423        assert_eq!(inflight, VecDeque::from(vec![12]));
5424
5425        handle_ack(&mut inflight, &mut last_ack, 8);
5426        assert_eq!(last_ack, 9, "ack lower than last should be ignored");
5427        assert_eq!(inflight, VecDeque::from(vec![12]));
5428
5429        handle_ack(&mut inflight, &mut last_ack, 15);
5430        assert_eq!(last_ack, 15);
5431        assert!(inflight.is_empty());
5432    }
5433
5434    #[test]
5435    fn passes_filters_applies_tags_and_severities() {
5436        let run_id = Uuid::new_v4();
5437        let step_id = Uuid::new_v4();
5438        let base_config = SubscriptionConfig {
5439            run_id,
5440            step_ids: Some(vec![step_id].into_iter().collect()),
5441            kinds: Some(vec!["step_succeeded".to_string()].into_iter().collect()),
5442            severities: Some(vec![EventSeverity::Info].into_iter().collect()),
5443            tags: Some(vec!["severity:info".to_string()].into_iter().collect()),
5444            cursor: None,
5445            since_offset: 0,
5446            batch_size: 16,
5447            max_inflight: 8,
5448            heartbeat: Duration::from_millis(1_000),
5449        };
5450
5451        let event = models::OutboxEvent {
5452            id: 1,
5453            run_id,
5454            step_id: Some(step_id),
5455            kind: "STEP_SUCCEEDED".to_string(),
5456            payload: json!({
5457                "status": "succeeded"
5458            }),
5459            created_at: Utc::now(),
5460            published_at: None,
5461        };
5462
5463        let severity = derive_severity(&event.kind, &event.payload);
5464        assert_eq!(severity, EventSeverity::Info);
5465        let tags = derive_tags(&event.kind, event.step_id, &event.payload, severity);
5466        assert!(passes_filters(&base_config, &event, severity, &tags));
5467
5468        let bad_config = SubscriptionConfig {
5469            tags: Some(vec!["severity:error".to_string()].into_iter().collect()),
5470            ..base_config
5471        };
5472        assert!(!passes_filters(&bad_config, &event, severity, &tags));
5473    }
5474}
5475
5476#[cfg(test)]
5477mod tests {
5478    use super::*;
5479    use chrono::Utc;
5480    use fleetforge_storage::{ObjectStoreConfig, StorageConfig};
5481    use fleetforge_telemetry::audit::Audit;
5482    use serde_json::json;
5483    use std::collections::HashMap;
5484    use testcontainers::clients::Cli;
5485    use testcontainers::images::postgres::Postgres;
5486    use tokio::time::{sleep, Duration};
5487    use uuid::Uuid;
5488
5489    fn build_request(dag: serde_json::Value) -> runtime::SubmitRunRequest {
5490        build_request_with_key(dag, "")
5491    }
5492
5493    fn build_request_with_key(
5494        dag: serde_json::Value,
5495        idempotency_key: impl Into<String>,
5496    ) -> runtime::SubmitRunRequest {
5497        runtime::SubmitRunRequest {
5498            spec: Some(runtime::RunSpec {
5499                dag_json: serde_json::to_string(&dag).unwrap(),
5500                inputs: None,
5501                seed: 123,
5502                labels: HashMap::new(),
5503            }),
5504            idempotency_key: idempotency_key.into(),
5505        }
5506    }
5507
5508    #[tokio::test]
5509    async fn submit_run_rejects_invalid_step_schema() {
5510        if std::fs::metadata("/var/run/docker.sock").is_err()
5511            && std::env::var("DOCKER_HOST").is_err()
5512        {
5513            eprintln!("Skipping API tests because Docker is unavailable");
5514            return;
5515        }
5516
5517        let docker = Cli::default();
5518        let container = docker.run(Postgres::default());
5519        let port = container.get_host_port_ipv4(5432);
5520        let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5521        sleep(Duration::from_millis(250)).await;
5522
5523        let storage = Arc::new(
5524            Storage::connect(StorageConfig {
5525                database_url: db_url,
5526                max_connections: 2,
5527                connect_timeout: Duration::from_secs(10),
5528                object_store: ObjectStoreConfig::InMemory,
5529            })
5530            .await
5531            .expect("storage connect"),
5532        );
5533        let auth = Arc::new(AuthHandle::disabled());
5534        let audit = Arc::new(Audit::new(storage.pool().clone()));
5535        let attestation_vault: Arc<dyn AttestationVault> =
5536            Arc::new(InMemoryAttestationVault::new());
5537        let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5538        let service = RuntimeApiService::new(
5539            storage,
5540            auth,
5541            audit,
5542            attestation_vault,
5543            transparency,
5544            false,
5545            TransparencyScope::Disabled,
5546        );
5547
5548        let dag = serde_json::json!({
5549            "steps": [
5550                {
5551                    "type": "tool",
5552                    "inputs": {}
5553                }
5554            ]
5555        });
5556
5557        let request = build_request(dag);
5558        let response = service.submit_run(Request::new(request)).await;
5559        assert!(response.is_err(), "expected schema validation failure");
5560        let status = response.err().unwrap();
5561        assert_eq!(status.code(), tonic::Code::InvalidArgument);
5562        assert!(status.message().contains("step schema validation failed"));
5563        drop(container);
5564    }
5565
5566    #[tokio::test]
5567    async fn submit_run_rejects_missing_seed() {
5568        if std::fs::metadata("/var/run/docker.sock").is_err()
5569            && std::env::var("DOCKER_HOST").is_err()
5570        {
5571            eprintln!("Skipping API tests because Docker is unavailable");
5572            return;
5573        }
5574
5575        let docker = Cli::default();
5576        let container = docker.run(Postgres::default());
5577        let port = container.get_host_port_ipv4(5432);
5578        let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5579        sleep(Duration::from_millis(250)).await;
5580
5581        let storage = Arc::new(
5582            Storage::connect(StorageConfig {
5583                database_url: db_url,
5584                max_connections: 2,
5585                connect_timeout: Duration::from_secs(10),
5586                object_store: ObjectStoreConfig::InMemory,
5587            })
5588            .await
5589            .expect("storage connect"),
5590        );
5591        let auth = Arc::new(AuthHandle::disabled());
5592        let audit = Arc::new(Audit::new(storage.pool().clone()));
5593        let attestation_vault: Arc<dyn AttestationVault> =
5594            Arc::new(InMemoryAttestationVault::new());
5595        let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5596        let service = RuntimeApiService::new(
5597            storage,
5598            auth,
5599            audit,
5600            attestation_vault,
5601            transparency,
5602            false,
5603            TransparencyScope::Disabled,
5604        );
5605
5606        let step_id = Uuid::new_v4();
5607        let dag = json!({
5608            "steps": [
5609                {
5610                    "id": step_id,
5611                    "type": "tool",
5612                    "inputs": {
5613                        "command": ["echo", "ok"]
5614                    }
5615                }
5616            ]
5617        });
5618
5619        let mut request = build_request(dag);
5620        if let Some(spec) = request.spec.as_mut() {
5621            spec.seed = 0;
5622        }
5623
5624        let response = service.submit_run(Request::new(request)).await;
5625        assert!(response.is_err(), "expected seed validation failure");
5626        let status = response.err().unwrap();
5627        assert_eq!(status.code(), tonic::Code::InvalidArgument);
5628        assert!(status.message().contains("seed must be a positive integer"));
5629
5630        drop(container);
5631    }
5632
5633    #[tokio::test]
5634    async fn submit_run_invalid_dag_variants() {
5635        if std::fs::metadata("/var/run/docker.sock").is_err()
5636            && std::env::var("DOCKER_HOST").is_err()
5637        {
5638            eprintln!("Skipping API tests because Docker is unavailable");
5639            return;
5640        }
5641
5642        let docker = Cli::default();
5643        let container = docker.run(Postgres::default());
5644        let port = container.get_host_port_ipv4(5432);
5645        let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5646        sleep(Duration::from_millis(250)).await;
5647
5648        let storage = Arc::new(
5649            Storage::connect(StorageConfig {
5650                database_url: db_url,
5651                max_connections: 2,
5652                connect_timeout: Duration::from_secs(10),
5653                object_store: ObjectStoreConfig::InMemory,
5654            })
5655            .await
5656            .expect("storage connect"),
5657        );
5658        let auth = Arc::new(AuthHandle::disabled());
5659        let audit = Arc::new(Audit::new(storage.pool().clone()));
5660        let attestation_vault: Arc<dyn AttestationVault> =
5661            Arc::new(InMemoryAttestationVault::new());
5662        let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5663        let service = RuntimeApiService::new(
5664            Arc::clone(&storage),
5665            Arc::clone(&auth),
5666            Arc::clone(&audit),
5667            Arc::clone(&attestation_vault),
5668            transparency,
5669            false,
5670            TransparencyScope::Disabled,
5671        );
5672
5673        let step_a = Uuid::new_v4();
5674        let step_b = Uuid::new_v4();
5675        let valid_step = json!({
5676            "id": step_a,
5677            "type": "tool",
5678            "inputs": {}
5679        });
5680
5681        let cases: Vec<(&str, serde_json::Value, &str)> = vec![
5682            (
5683                "missing_steps",
5684                json!({ "edges": [] }),
5685                "dag_json.steps must be an array",
5686            ),
5687            (
5688                "unknown_step_type",
5689                json!({ "steps": [ { "id": step_a, "type": "unknown", "inputs": {} } ] }),
5690                "step schema validation failed",
5691            ),
5692            (
5693                "duplicate_step_id",
5694                json!({ "steps": [valid_step.clone(), valid_step.clone()] }),
5695                "duplicate step.id",
5696            ),
5697            (
5698                "self_edge",
5699                json!({ "steps": [valid_step.clone()], "edges": [[step_a, step_a]] }),
5700                "edge cannot reference the same step twice",
5701            ),
5702            (
5703                "edge_unknown_target",
5704                json!({ "steps": [valid_step.clone()], "edges": [[step_a, step_b]] }),
5705                "edge references unknown step id",
5706            ),
5707            (
5708                "duplicate_edge",
5709                json!({ "steps": [valid_step.clone(), {"id": step_b, "type": "tool", "inputs": {}}], "edges": [[step_a, step_b], [step_a, step_b]] }),
5710                "duplicate edge detected",
5711            ),
5712        ];
5713
5714        for (label, dag, expected) in cases {
5715            let request = build_request(dag);
5716            let response = service.submit_run(Request::new(request)).await;
5717            assert!(response.is_err(), "case {label} unexpectedly succeeded");
5718            let status = response.err().unwrap();
5719            assert_eq!(status.code(), tonic::Code::InvalidArgument, "case {label}");
5720            assert!(
5721                status.message().contains(expected),
5722                "case {label} expected '{expected}' but got '{}'",
5723                status.message()
5724            );
5725        }
5726
5727        drop(container);
5728    }
5729
5730    #[tokio::test]
5731    async fn submit_run_accepts_valid_dag_and_persists_steps() {
5732        if std::fs::metadata("/var/run/docker.sock").is_err()
5733            && std::env::var("DOCKER_HOST").is_err()
5734        {
5735            eprintln!("Skipping API tests because Docker is unavailable");
5736            return;
5737        }
5738
5739        let docker = Cli::default();
5740        let container = docker.run(Postgres::default());
5741        let port = container.get_host_port_ipv4(5432);
5742        let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5743        sleep(Duration::from_millis(250)).await;
5744
5745        let storage = Arc::new(
5746            Storage::connect(StorageConfig {
5747                database_url: db_url,
5748                max_connections: 2,
5749                connect_timeout: Duration::from_secs(10),
5750                object_store: ObjectStoreConfig::InMemory,
5751            })
5752            .await
5753            .expect("storage connect"),
5754        );
5755        let auth = Arc::new(AuthHandle::disabled());
5756        let audit = Arc::new(Audit::new(storage.pool().clone()));
5757        let attestation_vault: Arc<dyn AttestationVault> =
5758            Arc::new(InMemoryAttestationVault::new());
5759        let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5760        let service = RuntimeApiService::new(
5761            Arc::clone(&storage),
5762            Arc::clone(&auth),
5763            Arc::clone(&audit),
5764            Arc::clone(&attestation_vault),
5765            transparency,
5766            false,
5767            TransparencyScope::Disabled,
5768        );
5769
5770        let step_a = Uuid::new_v4();
5771        let step_b = Uuid::new_v4();
5772        let step_c = Uuid::new_v4();
5773
5774        let dag = serde_json::json!({
5775            "steps": [
5776                {
5777                    "id": step_a,
5778                    "type": "tool",
5779                    "inputs": {},
5780                    "policy": {"priority": 5}
5781                },
5782                {
5783                    "id": step_b,
5784                    "type": "llm",
5785                    "inputs": {},
5786                    "policy": {"priority": 3}
5787                },
5788                {
5789                    "id": step_c,
5790                    "type": "reduce",
5791                    "inputs": {},
5792                    "policy": {"priority": 1}
5793                }
5794            ],
5795            "edges": [
5796                [step_a, step_b],
5797                [step_b, step_c]
5798            ]
5799        });
5800
5801        let request = build_request(dag);
5802        let response = service
5803            .submit_run(Request::new(request))
5804            .await
5805            .expect("valid run should succeed")
5806            .into_inner();
5807
5808        let run_uuid = Uuid::parse_str(&response.run_id).expect("run_id must be UUID");
5809        let persisted_run = storage
5810            .fetch_run(run_uuid)
5811            .await
5812            .expect("fetch run")
5813            .expect("run should exist");
5814        assert_eq!(persisted_run.status, models::RunStatus::Pending);
5815
5816        let steps = storage
5817            .fetch_steps_for_run(run_uuid)
5818            .await
5819            .expect("fetch steps");
5820        assert_eq!(steps.len(), 3);
5821
5822        let mut by_idx = steps;
5823        by_idx.sort_by_key(|s| s.idx);
5824
5825        assert_eq!(by_idx[0].step_id, step_a);
5826        assert_eq!(by_idx[0].status, models::StepStatus::Queued);
5827        assert_eq!(by_idx[0].priority, 5);
5828        assert_eq!(by_idx[0].pending_dependencies, 0);
5829
5830        assert_eq!(by_idx[1].step_id, step_b);
5831        assert_eq!(by_idx[1].status, models::StepStatus::Blocked);
5832        assert_eq!(by_idx[1].priority, 3);
5833        assert_eq!(by_idx[1].pending_dependencies, 1);
5834
5835        assert_eq!(by_idx[2].step_id, step_c);
5836        assert_eq!(by_idx[2].status, models::StepStatus::Blocked);
5837        assert_eq!(by_idx[2].priority, 1);
5838        assert_eq!(by_idx[2].pending_dependencies, 1);
5839        assert!(
5840            by_idx
5841                .iter()
5842                .all(|s| s.total_dependencies >= s.pending_dependencies),
5843            "dependency counters should be consistent"
5844        );
5845        drop(container);
5846    }
5847
5848    #[tokio::test]
5849    async fn submit_run_respects_idempotency_key() {
5850        if std::fs::metadata("/var/run/docker.sock").is_err()
5851            && std::env::var("DOCKER_HOST").is_err()
5852        {
5853            eprintln!("Skipping API tests because Docker is unavailable");
5854            return;
5855        }
5856
5857        let docker = Cli::default();
5858        let container = docker.run(Postgres::default());
5859        let port = container.get_host_port_ipv4(5432);
5860        let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5861        sleep(Duration::from_millis(250)).await;
5862
5863        let storage = Arc::new(
5864            Storage::connect(StorageConfig {
5865                database_url: db_url,
5866                max_connections: 2,
5867                connect_timeout: Duration::from_secs(10),
5868                object_store: ObjectStoreConfig::InMemory,
5869            })
5870            .await
5871            .expect("storage connect"),
5872        );
5873        let auth = Arc::new(AuthHandle::disabled());
5874        let audit = Arc::new(Audit::new(storage.pool().clone()));
5875        let attestation_vault: Arc<dyn AttestationVault> =
5876            Arc::new(InMemoryAttestationVault::new());
5877        let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5878        let service = RuntimeApiService::new(
5879            Arc::clone(&storage),
5880            Arc::clone(&auth),
5881            Arc::clone(&audit),
5882            Arc::clone(&attestation_vault),
5883            transparency,
5884            false,
5885            TransparencyScope::Disabled,
5886        );
5887
5888        let step_id = Uuid::new_v4();
5889        let dag = serde_json::json!({
5890            "steps": [{
5891                "id": step_id,
5892                "type": "tool",
5893                "inputs": {}
5894            }]
5895        });
5896        let raw_key = "  idempotent-key  ";
5897        let normalized_key = "idempotent-key";
5898
5899        let first = service
5900            .submit_run(Request::new(build_request_with_key(dag.clone(), raw_key)))
5901            .await
5902            .expect("first submission")
5903            .into_inner();
5904
5905        let second = service
5906            .submit_run(Request::new(build_request_with_key(dag, raw_key)))
5907            .await
5908            .expect("second submission")
5909            .into_inner();
5910
5911        assert_eq!(
5912            first.run_id, second.run_id,
5913            "idempotent submissions should reuse run_id"
5914        );
5915
5916        let stored = storage
5917            .fetch_run_by_idempotency_key(normalized_key)
5918            .await
5919            .expect("lookup idempotent run")
5920            .expect("run should exist for idempotency key");
5921        assert_eq!(stored.run_id.to_string(), first.run_id);
5922
5923        drop(container);
5924    }
5925
5926    #[tokio::test]
5927    async fn get_attestations_returns_records() {
5928        if std::fs::metadata("/var/run/docker.sock").is_err()
5929            && std::env::var("DOCKER_HOST").is_err()
5930        {
5931            eprintln!("Skipping API tests because Docker is unavailable");
5932            return;
5933        }
5934
5935        let docker = Cli::default();
5936        let container = docker.run(Postgres::default());
5937        let port = container.get_host_port_ipv4(5432);
5938        let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5939        sleep(Duration::from_millis(250)).await;
5940
5941        let storage = Arc::new(
5942            Storage::connect(StorageConfig {
5943                database_url: db_url,
5944                max_connections: 2,
5945                connect_timeout: Duration::from_secs(10),
5946                object_store: ObjectStoreConfig::InMemory,
5947            })
5948            .await
5949            .expect("storage connect"),
5950        );
5951        let auth = Arc::new(AuthHandle::disabled());
5952        let audit = Arc::new(Audit::new(storage.pool().clone()));
5953        let attestation_vault: Arc<dyn AttestationVault> =
5954            Arc::new(InMemoryAttestationVault::new());
5955        let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5956        let service = RuntimeApiService::new(
5957            Arc::clone(&storage),
5958            Arc::clone(&auth),
5959            Arc::clone(&audit),
5960            Arc::clone(&attestation_vault),
5961            transparency,
5962            false,
5963            TransparencyScope::Disabled,
5964        );
5965
5966        let run_id = Uuid::new_v4();
5967        let attestation =
5968            Attestation::new(Uuid::new_v4(), Utc::now()).with_subject(TrustSubject::Run { run_id });
5969        attestation_vault
5970            .record(attestation.clone())
5971            .await
5972            .expect("record attestation");
5973
5974        let response = service
5975            .get_attestations(Request::new(runtime::GetAttestationsRequest {
5976                attestation_ids: vec![
5977                    attestation.id.to_string(),
5978                    attestation.id.to_string(),
5979                    Uuid::new_v4().to_string(),
5980                ],
5981            }))
5982            .await
5983            .expect("get_attestations RPC")
5984            .into_inner();
5985
5986        assert_eq!(response.attestations.len(), 1);
5987        let returned = &response.attestations[0];
5988        assert_eq!(returned.attestation_id, attestation.id.to_string());
5989        match returned
5990            .subject
5991            .as_ref()
5992            .and_then(|subject| subject.kind.as_ref())
5993        {
5994            Some(runtime::attestation_subject::Kind::Run(run_subject)) => {
5995                assert_eq!(run_subject.run_id, run_id.to_string());
5996            }
5997            other => panic!("unexpected subject {other:?}"),
5998        }
5999
6000        drop(container);
6001    }
6002
6003    #[test]
6004    fn compute_drift_reports_within_tolerance() {
6005        let recorded = json!({
6006            "tokens": 10,
6007            "nested": { "value": "ok" }
6008        });
6009        let current = recorded.clone();
6010
6011        let result = compute_drift(
6012            &recorded,
6013            &current,
6014            DEFAULT_TOKEN_TOLERANCE,
6015            DEFAULT_COST_TOLERANCE,
6016        );
6017        assert!(result.within_tolerance);
6018        assert_eq!(result.token_delta, Some(0.0));
6019        assert_eq!(result.cost_delta, None);
6020    }
6021
6022    #[test]
6023    fn compute_drift_flags_token_delta() {
6024        let recorded = json!({
6025            "tokens": 10,
6026            "cost": 1.0,
6027            "payload": { "message": "hello" }
6028        });
6029        let current = json!({
6030            "tokens": 15,
6031            "cost": 1.0,
6032            "payload": { "message": "hello" }
6033        });
6034
6035        let result = compute_drift(
6036            &recorded,
6037            &current,
6038            DEFAULT_TOKEN_TOLERANCE,
6039            DEFAULT_COST_TOLERANCE,
6040        );
6041        assert!(!result.within_tolerance);
6042        assert_eq!(result.token_delta, Some(5.0));
6043        assert_eq!(result.cost_delta, Some(0.0));
6044    }
6045}