1mod auth;
4mod http;
5mod transparency;
6mod transparency_writer;
7
8use std::collections::{HashMap, HashSet, VecDeque};
9use std::convert::{Infallible, TryFrom};
10use std::env;
11use std::net::SocketAddr;
12use std::sync::Arc;
13
14use ::http::{header, HeaderValue, Method};
15use anyhow::{anyhow, Context, Result};
16use auth::{AuthError, AuthHandle, Role};
17use bytes::Bytes;
18use chrono::{DateTime, TimeZone, Utc};
19use clickhouse::{Client as ClickhouseClient, Row};
20use fleetforge_changeops::{
21 BudgetSignal as GateBudgetSignal, ChangeDecisionEffect as GateEffect, ChangeDiff as GateDiff,
22 ChangeFollowupRecord, ChangeFollowupRequest as GateFollowupRequest,
23 ChangeGateDecision as GateDecision, ChangeGateEngine, ChangeGateRequest as GateRequest,
24 CoverageComponent as GateCoverageComponent, CoverageReport as GateCoverageReport,
25 DecisionScorecard as GateScorecard, EvalMetric as GateEvalMetric,
26 FollowupAction as GateFollowupAction, FollowupOutcome as GateFollowupOutcome,
27 MetricDirection as GateMetricDirection,
28};
29use fleetforge_common::{
30 labels::{labels_map_to_value, labels_value_to_map},
31 licensing::{feature_allowed, LicensedFeature},
32 prost_json::{json_to_prost_struct, json_to_prost_value, prost_struct_to_json},
33 validation::{validate_run_spec, validate_step_spec, SchemaValidationError},
34};
35use fleetforge_contracts::{
36 runtime,
37 tap::{
38 self, subscribe_run_request, EventSeverity, PolicyDecisionEffect, QueryEventSource,
39 QueryEventsRequest, QueryEventsResponse, RunEventEnvelope, RunTraceContext,
40 SubscribeRunAck, SubscribeRunOpen, SubscribeRunRequest, SubscribeRunResponse,
41 },
42 RuntimeService, RuntimeServiceClient, RuntimeServiceServer, TapService, TapServiceClient,
43 TapServiceServer,
44};
45use fleetforge_storage::{models, models::NewOutboxEvent, Storage};
46use fleetforge_telemetry::audit::Audit;
47use fleetforge_trust::{
48 Attestation, AttestationVault, InMemoryAttestationVault, ObjectStoreAttestationVault,
49 TrustSubject,
50};
51use futures::Stream;
52use http_body::Body as _;
53use hyper::service::{make_service_fn, service_fn};
54use hyper::{Body, Request as HyperRequest, Server as HyperServer};
55use serde::Deserialize;
56use serde_json::{json, Map, Number, Value};
57use std::pin::Pin;
58use tokio::net::TcpListener;
59use tokio::sync::mpsc;
60use tokio::time::{interval, sleep, Duration, MissedTickBehavior};
61use tokio_stream::wrappers::ReceiverStream;
62use tonic::transport::Server;
63use tonic::{Request, Response, Status};
64use tonic_health::pb::health_check_response::ServingStatus;
65use tonic_health::server::health_reporter;
66use tonic_reflection::server::Builder as ReflectionBuilder;
67use tower::util::BoxCloneService;
68use tower::{Service, ServiceExt};
69use tower_http::cors::CorsLayer;
70use tracing::{error, info, instrument, warn};
71use transparency::{transparency_log_from_env, TransparencyContext, TransparencyLog};
72use transparency_writer::TransparencyWriter;
73use uuid::Uuid;
74
75#[derive(Clone, Debug)]
76pub struct RuntimeMetadata {
77 pub commit_sha: String,
78 pub policy_pack_id: String,
79 pub policy_pack_digest: String,
80 pub trust_mesh_alpha: bool,
81}
82
83impl RuntimeMetadata {
84 pub fn new(
85 commit_sha: impl Into<String>,
86 policy_pack_id: impl Into<String>,
87 policy_pack_digest: impl Into<String>,
88 trust_mesh_alpha: bool,
89 ) -> Self {
90 Self {
91 commit_sha: commit_sha.into(),
92 policy_pack_id: policy_pack_id.into(),
93 policy_pack_digest: policy_pack_digest.into(),
94 trust_mesh_alpha,
95 }
96 }
97}
98
99#[derive(Clone, Copy, Debug, PartialEq, Eq)]
100enum TransparencyScope {
101 Disabled,
102 Gates,
103 Runs,
104 Artifacts,
105}
106
107impl TransparencyScope {
108 fn includes_gates(self) -> bool {
109 matches!(
110 self,
111 TransparencyScope::Gates | TransparencyScope::Runs | TransparencyScope::Artifacts
112 )
113 }
114}
115#[derive(Clone)]
116struct RuntimeApiService {
117 storage: Arc<Storage>,
118 auth: Arc<AuthHandle>,
119 audit: Arc<Audit>,
120 attestation_vault: Arc<dyn AttestationVault>,
121 changeops: ChangeGateEngine,
122 transparency: Arc<dyn TransparencyLog>,
123 transparency_writer_enabled: bool,
124 transparency_scope: TransparencyScope,
125}
126
127impl RuntimeApiService {
128 fn new(
129 storage: Arc<Storage>,
130 auth: Arc<AuthHandle>,
131 audit: Arc<Audit>,
132 attestation_vault: Arc<dyn AttestationVault>,
133 transparency: Arc<dyn TransparencyLog>,
134 transparency_writer_enabled: bool,
135 transparency_scope: TransparencyScope,
136 ) -> Self {
137 Self {
138 storage,
139 auth,
140 audit,
141 attestation_vault,
142 changeops: ChangeGateEngine::default(),
143 transparency,
144 transparency_writer_enabled,
145 transparency_scope,
146 }
147 }
148
149 async fn require_role(
150 &self,
151 metadata: &tonic::metadata::MetadataMap,
152 required: Role,
153 ) -> Result<Role, Status> {
154 self.auth
155 .require_grpc(metadata, required)
156 .await
157 .map_err(|err| match err {
158 AuthError::MissingToken | AuthError::InvalidToken => {
159 Status::unauthenticated(err.to_string())
160 }
161 AuthError::InsufficientRole { .. }
162 | AuthError::MissingRoleClaim(_)
163 | AuthError::UnsupportedRole { .. } => Status::permission_denied(err.to_string()),
164 AuthError::NotConfigured => Status::unauthenticated(err.to_string()),
165 AuthError::Backend(_) => Status::internal(err.to_string()),
166 })
167 }
168
169 fn actor_from_metadata(metadata: &tonic::metadata::MetadataMap, role: Role) -> String {
170 metadata
171 .get("x-actor")
172 .and_then(|value| value.to_str().ok())
173 .map(|value| value.to_string())
174 .unwrap_or_else(|| format!("role:{role:?}"))
175 }
176}
177
178fn invalid_argument(message: impl Into<String>) -> Status {
179 Status::invalid_argument(message.into())
180}
181
182fn internal_error(message: impl Into<String>) -> Status {
183 Status::internal(message.into())
184}
185
186const QUERY_DEFAULT_PAGE_SIZE: u32 = 100;
187const QUERY_MAX_PAGE_SIZE: u32 = 500;
188const QUERY_FETCH_MULTIPLIER: u32 = 4;
189const TRANSPARENCY_WRITER_FLAG: &str = "FLEETFORGE_TRANSPARENCY_WRITER";
190const TRANSPARENCY_WRITER_INTERVAL_ENV: &str = "FLEETFORGE_TRANSPARENCY_WRITER_INTERVAL_SECS";
191const TRANSPARENCY_WRITER_INTERVAL_MIN_SECS: u64 = 5;
192const TRANSPARENCY_WRITER_DEFAULT_INTERVAL_SECS: u64 = 30;
193
194fn to_timestamp(time: DateTime<Utc>) -> prost_types::Timestamp {
195 prost_types::Timestamp {
196 seconds: time.timestamp(),
197 nanos: time.timestamp_subsec_nanos() as i32,
198 }
199}
200
201fn value_to_struct(value: &Value) -> Option<prost_types::Struct> {
202 if value.is_null() {
203 return None;
204 }
205 match json_to_prost_struct(value) {
206 Ok(struct_value) => Some(struct_value),
207 Err(err) => {
208 warn!(error = %err, "failed to convert JSON value to protobuf Struct");
209 None
210 }
211 }
212}
213
214impl RuntimeApiService {
215 async fn publish_scitt_entry(
216 &self,
217 change_id: &str,
218 gate_id: Uuid,
219 artifact_sha: &str,
220 attestation_refs: &[Uuid],
221 metadata: &mut Value,
222 ) -> Result<(), Status> {
223 let signer = fleetforge_trust::scitt_signer()
224 .map_err(|err| internal_error(format!("SCITT signer unavailable: {err}")))?;
225 let scitt_entry = fleetforge_trust::build_scitt_entry(
226 change_id,
227 attestation_refs,
228 artifact_sha,
229 metadata,
230 signer.as_ref(),
231 )
232 .map_err(|err| internal_error(format!("failed to build SCITT entry: {err}")))?;
233
234 let transparency_ctx = TransparencyContext {
235 change_id,
236 gate_id: &gate_id,
237 artifact_sha256: artifact_sha,
238 attestation_ids: attestation_refs,
239 };
240 let record = self
241 .transparency
242 .append(&scitt_entry, &transparency_ctx)
243 .await
244 .map_err(|err| internal_error(format!("failed to publish SCITT entry: {err}")))?;
245
246 let scitt_metadata = json!({
247 "id": record.entry_id,
248 "sha256": hex::encode(record.artifact.sha256),
249 "media_type": record.artifact.media_type.clone(),
250 "size": record.artifact.size,
251 "attestation_ids": attestation_refs.iter().map(|id| id.to_string()).collect::<Vec<_>>(),
252 "transparency": record.receipt,
253 });
254
255 if let Value::Object(mut meta_obj) = metadata.take() {
256 meta_obj.insert("scitt_entry".to_string(), scitt_metadata);
257 *metadata = Value::Object(meta_obj);
258 } else {
259 *metadata = json!({ "scitt_entry": scitt_metadata });
260 }
261 Ok(())
262 }
263}
264
265fn struct_to_value(data: Option<prost_types::Struct>) -> Value {
266 data.as_ref()
267 .map(prost_struct_to_json)
268 .unwrap_or(Value::Null)
269}
270
271fn attestation_to_proto(att: Attestation) -> Result<runtime::Attestation, Status> {
272 let subject = att
273 .subject
274 .as_ref()
275 .map(attestation_subject_to_proto)
276 .transpose()?;
277
278 let claims_value = Value::Object(att.claims.into_iter().collect());
279 let claims_struct = json_to_prost_struct(&claims_value)
280 .map_err(|err| internal_error(format!("failed to serialise attestation claims: {err}")))?;
281
282 Ok(runtime::Attestation {
283 attestation_id: att.id.to_string(),
284 issued_at: Some(to_timestamp(att.issued_at)),
285 subject,
286 artifact_hash: att.artifact_hash.map(|value| value.into()),
287 claims: Some(claims_struct),
288 signature: att.signature.map(|value| value.into()),
289 })
290}
291
292fn attestation_subject_to_proto(
293 subject: &TrustSubject,
294) -> Result<runtime::AttestationSubject, Status> {
295 use runtime::attestation_subject::Kind;
296 let kind = match subject {
297 TrustSubject::Run { run_id } => Kind::Run(runtime::AttestationRunSubject {
298 run_id: run_id.to_string(),
299 }),
300 TrustSubject::Step { run_id, step_id } => Kind::Step(runtime::AttestationStepSubject {
301 run_id: run_id.to_string(),
302 step_id: step_id.to_string(),
303 }),
304 TrustSubject::Tool { name } => {
305 Kind::Tool(runtime::AttestationToolSubject { name: name.clone() })
306 }
307 TrustSubject::Artifact { uri } => {
308 Kind::Artifact(runtime::AttestationArtifactSubject { uri: uri.clone() })
309 }
310 TrustSubject::CapabilityToken { token_id } => {
311 Kind::CapabilityToken(runtime::AttestationCapabilitySubject {
312 token_id: token_id.to_string(),
313 })
314 }
315 TrustSubject::Custom { label } => Kind::Custom(runtime::AttestationCustomSubject {
316 label: label.clone(),
317 }),
318 };
319 Ok(runtime::AttestationSubject { kind: Some(kind) })
320}
321
322fn gate_effect_to_proto(effect: GateEffect) -> runtime::ChangeDecisionEffect {
323 match effect {
324 GateEffect::Allow => runtime::ChangeDecisionEffect::Allow,
325 GateEffect::FollowUp => runtime::ChangeDecisionEffect::FollowUp,
326 GateEffect::Deny => runtime::ChangeDecisionEffect::Deny,
327 }
328}
329
330fn gate_effect_from_str(value: &str) -> GateEffect {
331 match value {
332 "allow" => GateEffect::Allow,
333 "deny" => GateEffect::Deny,
334 "follow_up" => GateEffect::FollowUp,
335 other => {
336 warn!(effect = %other, "unknown change gate effect stored; defaulting to follow_up");
337 GateEffect::FollowUp
338 }
339 }
340}
341
342fn gate_followup_action_to_proto(action: &GateFollowupAction) -> runtime::FollowupAction {
343 runtime::FollowupAction {
344 slug: action.slug.clone(),
345 description: action.description.clone(),
346 recommended: action.recommended,
347 }
348}
349
350fn gate_scorecard_to_proto(scorecard: &GateScorecard) -> Option<runtime::DecisionScorecard> {
351 Some(runtime::DecisionScorecard {
352 novelty: Some(runtime::NoveltySummary {
353 max_novelty: scorecard.novelty.max_novelty,
354 avg_novelty: scorecard.novelty.avg_novelty,
355 high_risk_paths: scorecard.novelty.high_risk_paths.clone(),
356 }),
357 coverage: Some(runtime::CoverageSummary {
358 overall_ratio: scorecard.coverage.overall_ratio,
359 components_below_threshold: scorecard.coverage.components_below_threshold.clone(),
360 }),
361 evals: Some(runtime::EvalSummary {
362 failing_metrics: scorecard.evals.failing_metrics.clone(),
363 attention_metrics: scorecard.evals.attention_metrics.clone(),
364 score_per_token: scorecard.evals.score_per_token,
365 }),
366 budgets: Some(runtime::BudgetSummary {
367 breaches: scorecard.budgets.breaches.clone(),
368 near_limits: scorecard.budgets.near_limits.clone(),
369 }),
370 telemetry: value_to_struct(&scorecard.telemetry),
371 })
372}
373
374fn proto_followup_outcome(outcome: runtime::FollowupOutcome) -> GateFollowupOutcome {
375 match outcome {
376 runtime::FollowupOutcome::Approved => GateFollowupOutcome::Approved,
377 runtime::FollowupOutcome::NeedsChanges => GateFollowupOutcome::NeedsChanges,
378 runtime::FollowupOutcome::Deferred => GateFollowupOutcome::Deferred,
379 runtime::FollowupOutcome::Unspecified => GateFollowupOutcome::Approved,
380 }
381}
382
383fn followup_outcome_to_proto(outcome: GateFollowupOutcome) -> runtime::FollowupOutcome {
384 match outcome {
385 GateFollowupOutcome::Approved => runtime::FollowupOutcome::Approved,
386 GateFollowupOutcome::NeedsChanges => runtime::FollowupOutcome::NeedsChanges,
387 GateFollowupOutcome::Deferred => runtime::FollowupOutcome::Deferred,
388 }
389}
390
391fn parse_reasons(value: &Value) -> Vec<String> {
392 serde_json::from_value::<Vec<String>>(value.clone()).unwrap_or_default()
393}
394
395fn parse_followup_actions(value: &Value) -> Vec<GateFollowupAction> {
396 serde_json::from_value::<Vec<GateFollowupAction>>(value.clone()).unwrap_or_default()
397}
398
399fn parse_scorecard(value: &Value) -> GateScorecard {
400 serde_json::from_value::<GateScorecard>(value.clone()).unwrap_or_default()
401}
402
403fn gate_followup_record_to_proto(
404 record: &models::ChangeGateFollowup,
405) -> runtime::ChangeGateFollowup {
406 let outcome =
407 GateFollowupOutcome::from_str(&record.outcome).unwrap_or(GateFollowupOutcome::Approved);
408 let details_struct = value_to_struct(&record.details);
409
410 runtime::ChangeGateFollowup {
411 followup_id: record.followup_id.to_string(),
412 gate_id: record.gate_id.to_string(),
413 actor: record.actor.clone(),
414 outcome: followup_outcome_to_proto(outcome) as i32,
415 note: record.note.clone(),
416 details: details_struct,
417 recorded_at: Some(to_timestamp(record.recorded_at)),
418 }
419}
420
421fn build_proto_change_gate(
422 gate_id: Uuid,
423 change_id: &str,
424 revision: Option<&str>,
425 decision: &GateDecision,
426 telemetry: &Value,
427 acknowledgements: &[models::ChangeGateFollowup],
428) -> Result<runtime::ChangeGateDecision, Status> {
429 let effect = gate_effect_to_proto(decision.effect) as i32;
430 let scorecard = gate_scorecard_to_proto(&decision.scorecard);
431 let metadata_struct = value_to_struct(&decision.metadata);
432 let telemetry_struct = value_to_struct(telemetry);
433 let ack_proto = acknowledgements
434 .iter()
435 .map(gate_followup_record_to_proto)
436 .collect();
437
438 Ok(runtime::ChangeGateDecision {
439 gate_id: gate_id.to_string(),
440 change_id: change_id.to_string(),
441 revision: revision.unwrap_or_default().to_string(),
442 effect,
443 reasons: decision.reasons.clone(),
444 followups: decision
445 .followups
446 .iter()
447 .map(gate_followup_action_to_proto)
448 .collect(),
449 scorecard,
450 decided_at: Some(to_timestamp(decision.decided_at)),
451 metadata: metadata_struct,
452 telemetry: telemetry_struct,
453 acknowledgements: ack_proto,
454 })
455}
456
457fn attestation_ids_from_value(value: &Value) -> Vec<Uuid> {
458 fn recurse(value: &Value, acc: &mut HashSet<Uuid>) {
459 match value {
460 Value::Object(map) => {
461 if let Some(ids) = map.get("attestation_ids").and_then(Value::as_array) {
462 for id in ids {
463 if let Some(s) = id.as_str() {
464 if let Ok(uuid) = Uuid::parse_str(s) {
465 acc.insert(uuid);
466 }
467 }
468 }
469 }
470 for v in map.values() {
471 recurse(v, acc);
472 }
473 }
474 Value::Array(items) => {
475 for item in items {
476 recurse(item, acc);
477 }
478 }
479 _ => {}
480 }
481 }
482 let mut set = HashSet::new();
483 recurse(value, &mut set);
484 set.into_iter().collect()
485}
486
487#[derive(Clone)]
488struct TapApiService {
489 storage: Arc<Storage>,
490 auth: Arc<AuthHandle>,
491 clickhouse: Option<ClickhouseEventQuery>,
492}
493
494impl TapApiService {
495 fn new(storage: Arc<Storage>, auth: Arc<AuthHandle>) -> Self {
496 let clickhouse = ClickhouseEventQuery::from_env().ok();
497 Self {
498 storage,
499 auth,
500 clickhouse,
501 }
502 }
503
504 async fn require_role(
505 &self,
506 metadata: &tonic::metadata::MetadataMap,
507 required: Role,
508 ) -> Result<Role, Status> {
509 self.auth
510 .require_grpc(metadata, required)
511 .await
512 .map_err(|err| match err {
513 AuthError::MissingToken | AuthError::InvalidToken => {
514 Status::unauthenticated(err.to_string())
515 }
516 AuthError::InsufficientRole { .. }
517 | AuthError::MissingRoleClaim(_)
518 | AuthError::UnsupportedRole { .. } => Status::permission_denied(err.to_string()),
519 AuthError::NotConfigured => Status::unauthenticated(err.to_string()),
520 AuthError::Backend(_) => Status::internal(err.to_string()),
521 })
522 }
523}
524
525#[tonic::async_trait]
526impl TapService for TapApiService {
527 type SubscribeRunStream =
528 Pin<Box<dyn Stream<Item = Result<SubscribeRunResponse, Status>> + Send>>;
529
530 async fn subscribe_run(
531 &self,
532 request: Request<tonic::Streaming<SubscribeRunRequest>>,
533 ) -> Result<Response<Self::SubscribeRunStream>, Status> {
534 let metadata = request.metadata().clone();
535 self.require_role(&metadata, Role::Reader).await?;
536
537 let mut inbound = request.into_inner();
538 let first = inbound
539 .message()
540 .await
541 .map_err(|status| internal_error(status.to_string()))?
542 .ok_or_else(|| invalid_argument("missing subscribe request"))?;
543
544 let open = match first.request {
545 Some(subscribe_run_request::Request::Open(open)) => open,
546 Some(subscribe_run_request::Request::Ack(_)) => {
547 return Err(invalid_argument(
548 "first SubscribeRun message must be `open`",
549 ))
550 }
551 None => return Err(invalid_argument("first SubscribeRun message must be set")),
552 };
553
554 let mut config = SubscriptionConfig::try_from_open(open)?;
555 if let Some(cursor) = config.cursor.clone() {
556 if config.since_offset == 0 {
557 match self.storage.fetch_tap_offset(config.run_id, &cursor).await {
558 Ok(Some(stored)) => {
559 config.since_offset = stored;
560 }
561 Ok(None) => {}
562 Err(err) => {
563 error!(
564 run = %config.run_id,
565 cursor = %cursor,
566 error = %err,
567 "failed to load stored tap cursor"
568 );
569 return Err(internal_error("failed to load stored tap cursor"));
570 }
571 }
572 }
573 }
574 let (tx, rx) =
575 mpsc::channel::<Result<SubscribeRunResponse, Status>>(config.max_inflight * 2);
576
577 let storage = Arc::clone(&self.storage);
578 tokio::spawn(async move {
579 if let Err(err) = run_subscription(storage, config, inbound, tx.clone()).await {
580 let _ = tx.send(Err(err)).await;
581 }
582 });
583
584 Ok(Response::new(
585 Box::pin(ReceiverStream::new(rx)) as Self::SubscribeRunStream
586 ))
587 }
588
589 async fn query_events(
590 &self,
591 request: Request<QueryEventsRequest>,
592 ) -> Result<Response<QueryEventsResponse>, Status> {
593 let metadata = request.metadata().clone();
594 self.require_role(&metadata, Role::Reader).await?;
595
596 let clickhouse = self.clickhouse.as_ref().ok_or_else(|| {
597 Status::failed_precondition("QueryEvents requires ClickHouse configuration")
598 })?;
599
600 let inner = request.into_inner();
601 let filters = QueryFiltersOwned::from_request(&inner)?;
602
603 let requested_page = if inner.page_size == 0 {
604 QUERY_DEFAULT_PAGE_SIZE
605 } else {
606 inner.page_size
607 };
608 let page_size = requested_page.clamp(1, QUERY_MAX_PAGE_SIZE);
609 let offset = parse_page_token(inner.page_token.as_str())?;
610
611 let QueryBatch {
612 events,
613 next_offset,
614 } = clickhouse
615 .fetch(&filters, offset, page_size)
616 .await
617 .map_err(|err| {
618 error!(error = %err, "failed to query ClickHouse events");
619 internal_error("failed to query ClickHouse events")
620 })?;
621
622 let next_page_token = next_offset
623 .map(|value| value.to_string())
624 .unwrap_or_default();
625
626 Ok(Response::new(QueryEventsResponse {
627 events,
628 next_page_token,
629 }))
630 }
631}
632
633struct SubscriptionConfig {
634 run_id: Uuid,
635 step_ids: Option<HashSet<Uuid>>,
636 kinds: Option<HashSet<String>>,
637 severities: Option<HashSet<EventSeverity>>,
638 tags: Option<HashSet<String>>,
639 cursor: Option<String>,
640 since_offset: i64,
641 batch_size: i64,
642 max_inflight: usize,
643 heartbeat: Duration,
644}
645
646impl SubscriptionConfig {
647 fn try_from_open(open: SubscribeRunOpen) -> Result<Self, Status> {
648 let SubscribeRunOpen {
649 run_ids,
650 step_ids,
651 kinds,
652 severities,
653 tags,
654 since_offset,
655 batch_size,
656 max_inflight,
657 heartbeat_interval_ms,
658 cursor,
659 ..
660 } = open;
661
662 if run_ids.is_empty() {
663 return Err(invalid_argument(
664 "subscribe_run requires at least one run_id",
665 ));
666 }
667 if run_ids.len() > 1 {
668 return Err(invalid_argument(
669 "subscribe_run currently supports a single run_id",
670 ));
671 }
672 let run_id = Uuid::parse_str(&run_ids[0])
673 .map_err(|err| invalid_argument(format!("invalid run_id '{}' : {err}", run_ids[0])))?;
674
675 let step_ids = if step_ids.is_empty() {
676 None
677 } else {
678 let mut ids = HashSet::with_capacity(step_ids.len());
679 for raw in step_ids {
680 let parsed = Uuid::parse_str(&raw)
681 .map_err(|err| invalid_argument(format!("invalid step_id '{raw}': {err}")))?;
682 ids.insert(parsed);
683 }
684 Some(ids)
685 };
686
687 let kinds = if kinds.is_empty() {
688 None
689 } else {
690 Some(
691 kinds
692 .into_iter()
693 .map(|kind| kind.to_ascii_lowercase())
694 .collect::<HashSet<_>>(),
695 )
696 };
697
698 let severities = if severities.is_empty() {
699 None
700 } else {
701 let mut set = HashSet::with_capacity(severities.len());
702 for raw in severities {
703 let sev = EventSeverity::try_from(raw)
704 .map_err(|_| invalid_argument(format!("unknown severity enum value {raw}")))?;
705 set.insert(sev);
706 }
707 Some(set)
708 };
709
710 let tags = if tags.is_empty() {
711 None
712 } else {
713 Some(
714 tags.into_iter()
715 .map(|tag| tag.to_ascii_lowercase())
716 .collect::<HashSet<_>>(),
717 )
718 };
719
720 if since_offset < 0 {
721 return Err(invalid_argument("since_offset must be >= 0"));
722 }
723
724 let batch_raw = if batch_size == 0 { 256 } else { batch_size };
725 let batch_size = batch_raw.max(1).min(2048) as i64;
726
727 let inflight_raw = if max_inflight == 0 { 128 } else { max_inflight };
728 let max_inflight = inflight_raw.max(1).min(4096) as usize;
729
730 let heartbeat_ms = if heartbeat_interval_ms == 0 {
731 1000u64
732 } else {
733 heartbeat_interval_ms as u64
734 }
735 .clamp(250, 60_000);
736
737 let cursor = {
738 let trimmed = cursor.trim();
739 if trimmed.is_empty() {
740 None
741 } else {
742 Some(trimmed.to_string())
743 }
744 };
745
746 Ok(Self {
747 run_id,
748 step_ids,
749 kinds,
750 severities,
751 tags,
752 cursor,
753 since_offset,
754 batch_size,
755 max_inflight,
756 heartbeat: Duration::from_millis(heartbeat_ms),
757 })
758 }
759}
760
761async fn run_subscription(
762 storage: Arc<Storage>,
763 config: SubscriptionConfig,
764 mut inbound: tonic::Streaming<SubscribeRunRequest>,
765 tx: mpsc::Sender<Result<SubscribeRunResponse, Status>>,
766) -> Result<(), Status> {
767 let mut last_seen = config.since_offset;
768 let mut last_ack = config.since_offset;
769 let mut inflight: VecDeque<i64> = VecDeque::new();
770 let mut ticker = interval(config.heartbeat);
771 ticker.set_missed_tick_behavior(MissedTickBehavior::Delay);
772
773 loop {
774 while inflight.len() < config.max_inflight {
775 let events = storage
776 .fetch_outbox_for_run_since(config.run_id, last_seen, config.batch_size)
777 .await
778 .map_err(|err| internal_error(format!("failed to fetch outbox events: {err}")))?;
779
780 if events.is_empty() {
781 break;
782 }
783
784 last_seen = events.last().map(|event| event.id).unwrap_or(last_seen);
785
786 for event in events {
787 last_seen = event.id;
788 let severity = derive_severity(&event.kind, &event.payload);
789 let tags = derive_tags(&event.kind, event.step_id, &event.payload, severity);
790 if !passes_filters(&config, &event, severity, &tags) {
791 continue;
792 }
793
794 let response = build_response(&config, &event, severity, &tags, last_seen)?;
795 if tx.send(Ok(response)).await.is_err() {
796 return Ok(());
797 }
798 inflight.push_back(event.id);
799 }
800 }
801
802 tokio::select! {
803 maybe_req = inbound.message() => {
804 match maybe_req {
805 Ok(Some(SubscribeRunRequest { request: Some(subscribe_run_request::Request::Ack(ack)) })) => {
806 handle_ack(&mut inflight, &mut last_ack, ack.offset);
807 let ack_cursor = if !ack.cursor.trim().is_empty() {
808 Some(ack.cursor)
809 } else {
810 config.cursor.clone()
811 };
812 if let Some(cursor_id) = ack_cursor {
813 if ack.offset < 0 {
814 return Err(invalid_argument("ack offset must be >= 0"));
815 }
816 if let Err(err) = storage
817 .upsert_tap_offset(config.run_id, &cursor_id, ack.offset)
818 .await
819 {
820 error!(
821 run = %config.run_id,
822 cursor = %cursor_id,
823 error = %err,
824 "failed to persist tap cursor offset"
825 );
826 return Err(internal_error("failed to persist tap cursor offset"));
827 }
828 }
829 }
830 Ok(Some(SubscribeRunRequest { request: Some(subscribe_run_request::Request::Open(_)) })) => {
831 return Err(invalid_argument("subscribe_run received multiple open messages"));
832 }
833 Ok(Some(SubscribeRunRequest { request: None })) => {}
834 Ok(None) => {
835 return Ok(());
836 }
837 Err(status) => {
838 return Err(internal_error(status.to_string()));
839 }
840 }
841 }
842 _ = ticker.tick() => {
843 let heartbeat = SubscribeRunResponse {
844 offset: last_ack,
845 watermark: last_seen,
846 heartbeat: true,
847 event: None,
848 };
849 if tx.send(Ok(heartbeat)).await.is_err() {
850 return Ok(());
851 }
852 }
853 }
854 }
855}
856
857#[derive(Debug, Clone)]
858struct QueryFiltersOwned {
859 source: QueryEventSource,
860 run_ids: Vec<Uuid>,
861 step_ids: Vec<Uuid>,
862 workspace_ids: Vec<String>,
863 kinds: Vec<String>,
864 policy_effects: Vec<PolicyDecisionEffect>,
865 severities: Vec<EventSeverity>,
866 start_time: Option<DateTime<Utc>>,
867 end_time: Option<DateTime<Utc>>,
868 ascending: bool,
869}
870
871impl QueryFiltersOwned {
872 fn from_request(request: &QueryEventsRequest) -> Result<Self, Status> {
873 let mut run_ids = Vec::with_capacity(request.run_ids.len());
874 for raw in &request.run_ids {
875 let parsed = Uuid::parse_str(raw)
876 .map_err(|err| invalid_argument(format!("invalid run_id '{raw}': {err}")))?;
877 run_ids.push(parsed);
878 }
879
880 let mut step_ids = Vec::with_capacity(request.step_ids.len());
881 for raw in &request.step_ids {
882 let parsed = Uuid::parse_str(raw)
883 .map_err(|err| invalid_argument(format!("invalid step_id '{raw}': {err}")))?;
884 step_ids.push(parsed);
885 }
886
887 let source = QueryEventSource::try_from(request.source).unwrap_or(QueryEventSource::Run);
888
889 let start_time = request
890 .start_time
891 .as_ref()
892 .map(timestamp_to_datetime)
893 .transpose()?;
894 let end_time = request
895 .end_time
896 .as_ref()
897 .map(timestamp_to_datetime)
898 .transpose()?;
899
900 let kinds = request
901 .kinds
902 .iter()
903 .filter(|kind| !kind.is_empty())
904 .map(|kind| kind.to_ascii_lowercase())
905 .collect::<Vec<_>>();
906
907 let workspace_ids = request
908 .workspace_ids
909 .iter()
910 .filter(|id| !id.is_empty())
911 .map(|id| id.to_ascii_lowercase())
912 .collect::<Vec<_>>();
913
914 let mut policy_effects = Vec::with_capacity(request.policy_effects.len());
915 for raw in &request.policy_effects {
916 if let Ok(effect) = PolicyDecisionEffect::try_from(*raw) {
917 if effect != PolicyDecisionEffect::Unspecified {
918 policy_effects.push(effect);
919 }
920 }
921 }
922
923 let mut severities = Vec::with_capacity(request.severities.len());
924 for raw in &request.severities {
925 if let Ok(severity) = EventSeverity::try_from(*raw) {
926 if severity != EventSeverity::Unspecified {
927 severities.push(severity);
928 }
929 }
930 }
931
932 Ok(Self {
933 source,
934 run_ids,
935 step_ids,
936 workspace_ids,
937 kinds,
938 policy_effects,
939 severities,
940 start_time,
941 end_time,
942 ascending: request.ascending,
943 })
944 }
945}
946
947struct QueryBatch {
948 events: Vec<RunEventEnvelope>,
949 next_offset: Option<u64>,
950}
951
952#[derive(Clone)]
953struct ClickhouseEventQuery {
954 client: ClickhouseClient,
955 run_table: String,
956 step_table: String,
957}
958
959#[derive(Debug, Deserialize, Row)]
960struct RunEventRow {
961 run_id: Uuid,
962 created_at: DateTime<Utc>,
963 status: String,
964 attributes: Value,
965}
966
967#[derive(Debug, Deserialize, Row)]
968struct StepEventRow {
969 run_id: Uuid,
970 step_id: Uuid,
971 status: String,
972 attempt: i32,
973 recorded_at: DateTime<Utc>,
974 attributes: Value,
975}
976
977impl ClickhouseEventQuery {
978 fn from_env() -> Result<Self> {
979 let url = env::var("CLICKHOUSE_DSN")
980 .context("CLICKHOUSE_DSN is not configured for Tap QueryEvents")?;
981 let database = env::var("CLICKHOUSE_DATABASE").unwrap_or_else(|_| "telemetry".to_string());
982 let run_table =
983 env::var("CLICKHOUSE_RUN_TABLE").unwrap_or_else(|_| "run_events".to_string());
984 let step_table =
985 env::var("CLICKHOUSE_STEP_TABLE").unwrap_or_else(|_| "step_events".to_string());
986
987 let client = ClickhouseClient::default()
988 .with_url(url)
989 .with_database(database);
990
991 Ok(Self {
992 client,
993 run_table,
994 step_table,
995 })
996 }
997
998 async fn fetch(
999 &self,
1000 filters: &QueryFiltersOwned,
1001 offset: u64,
1002 limit: u32,
1003 ) -> Result<QueryBatch> {
1004 match filters.source {
1005 QueryEventSource::Step => self.query_step_events(filters, offset, limit).await,
1006 _ => self.query_run_events(filters, offset, limit).await,
1007 }
1008 }
1009
1010 async fn query_run_events(
1011 &self,
1012 filters: &QueryFiltersOwned,
1013 offset: u64,
1014 limit: u32,
1015 ) -> Result<QueryBatch> {
1016 let chunk = std::cmp::max(limit, 1)
1017 .saturating_mul(QUERY_FETCH_MULTIPLIER)
1018 .max(limit.max(1));
1019 let mut collected = Vec::with_capacity(limit as usize);
1020 let mut scan_offset = offset;
1021 let mut next_offset = None;
1022
1023 while collected.len() < limit as usize {
1024 let rows = self.fetch_run_rows(filters, scan_offset, chunk).await?;
1025 if rows.is_empty() {
1026 break;
1027 }
1028
1029 let mut consumed = 0usize;
1030 for row in rows {
1031 let raw_offset = scan_offset + consumed as u64;
1032 consumed += 1;
1033 let payload = normalize_payload(row.attributes);
1034 let kind = format!("run_{}", row.status.to_ascii_lowercase());
1035 let severity = derive_severity(&kind, &payload);
1036 if !matches_filters(filters, &payload, &kind, severity, None) {
1037 continue;
1038 }
1039 let envelope = make_envelope(
1040 row.run_id,
1041 None,
1042 row.created_at,
1043 &payload,
1044 &kind,
1045 severity,
1046 raw_offset,
1047 )?;
1048 collected.push(envelope);
1049 if collected.len() >= limit as usize {
1050 next_offset = Some(raw_offset.saturating_add(1));
1051 break;
1052 }
1053 }
1054
1055 scan_offset = scan_offset.saturating_add(consumed as u64);
1056 if consumed < chunk as usize {
1057 break;
1058 }
1059 }
1060
1061 Ok(QueryBatch {
1062 events: collected,
1063 next_offset,
1064 })
1065 }
1066
1067 async fn query_step_events(
1068 &self,
1069 filters: &QueryFiltersOwned,
1070 offset: u64,
1071 limit: u32,
1072 ) -> Result<QueryBatch> {
1073 let chunk = std::cmp::max(limit, 1)
1074 .saturating_mul(QUERY_FETCH_MULTIPLIER)
1075 .max(limit.max(1));
1076 let mut collected = Vec::with_capacity(limit as usize);
1077 let mut scan_offset = offset;
1078 let mut next_offset = None;
1079
1080 while collected.len() < limit as usize {
1081 let rows = self.fetch_step_rows(filters, scan_offset, chunk).await?;
1082 if rows.is_empty() {
1083 break;
1084 }
1085
1086 let mut consumed = 0usize;
1087 for row in rows {
1088 let raw_offset = scan_offset + consumed as u64;
1089 consumed += 1;
1090 let payload = normalize_payload(row.attributes);
1091 let kind = format!("step_{}", row.status.to_ascii_lowercase());
1092 let severity = derive_severity(&kind, &payload);
1093 if !matches_filters(filters, &payload, &kind, severity, Some(row.step_id)) {
1094 continue;
1095 }
1096 let envelope = make_envelope(
1097 row.run_id,
1098 Some(row.step_id),
1099 row.recorded_at,
1100 &payload,
1101 &kind,
1102 severity,
1103 raw_offset,
1104 )?;
1105 collected.push(envelope);
1106 if collected.len() >= limit as usize {
1107 next_offset = Some(raw_offset.saturating_add(1));
1108 break;
1109 }
1110 }
1111
1112 scan_offset = scan_offset.saturating_add(consumed as u64);
1113 if consumed < chunk as usize {
1114 break;
1115 }
1116 }
1117
1118 Ok(QueryBatch {
1119 events: collected,
1120 next_offset,
1121 })
1122 }
1123
1124 async fn fetch_run_rows(
1125 &self,
1126 filters: &QueryFiltersOwned,
1127 offset: u64,
1128 limit: u32,
1129 ) -> Result<Vec<RunEventRow>> {
1130 let mut where_clauses = Vec::new();
1131 if !filters.run_ids.is_empty() {
1132 where_clauses.push(format!(
1133 "run_id IN ({})",
1134 format_uuid_list(&filters.run_ids)
1135 ));
1136 }
1137 if let Some(start) = filters.start_time {
1138 where_clauses.push(format!(
1139 "created_at >= toDateTime64('{}', 3, 'UTC')",
1140 start.format("%Y-%m-%d %H:%M:%S%.3f")
1141 ));
1142 }
1143 if let Some(end) = filters.end_time {
1144 where_clauses.push(format!(
1145 "created_at <= toDateTime64('{}', 3, 'UTC')",
1146 end.format("%Y-%m-%d %H:%M:%S%.3f")
1147 ));
1148 }
1149
1150 let mut sql = format!(
1151 "SELECT run_id, created_at, status, attributes FROM {}",
1152 self.run_table
1153 );
1154 if !where_clauses.is_empty() {
1155 sql.push_str(" WHERE ");
1156 sql.push_str(&where_clauses.join(" AND "));
1157 }
1158 sql.push_str(" ORDER BY created_at ");
1159 sql.push_str(if filters.ascending { "ASC" } else { "DESC" });
1160 sql.push_str(", run_id ");
1161 sql.push_str(if filters.ascending { "ASC" } else { "DESC" });
1162 sql.push_str(&format!(" LIMIT {} OFFSET {}", limit, offset));
1163
1164 let rows = self
1165 .client
1166 .query(&sql)
1167 .fetch_all::<RunEventRow>()
1168 .await
1169 .context("failed to fetch run events from ClickHouse")?;
1170 Ok(rows)
1171 }
1172
1173 async fn fetch_step_rows(
1174 &self,
1175 filters: &QueryFiltersOwned,
1176 offset: u64,
1177 limit: u32,
1178 ) -> Result<Vec<StepEventRow>> {
1179 let mut where_clauses = Vec::new();
1180 if !filters.run_ids.is_empty() {
1181 where_clauses.push(format!(
1182 "run_id IN ({})",
1183 format_uuid_list(&filters.run_ids)
1184 ));
1185 }
1186 if !filters.step_ids.is_empty() {
1187 where_clauses.push(format!(
1188 "step_id IN ({})",
1189 format_uuid_list(&filters.step_ids)
1190 ));
1191 }
1192
1193 let time_expr = "coalesce(finished_at, started_at)";
1194 if let Some(start) = filters.start_time {
1195 where_clauses.push(format!(
1196 "{time_expr} >= toDateTime64('{}', 3, 'UTC')",
1197 start.format("%Y-%m-%d %H:%M:%S%.3f")
1198 ));
1199 }
1200 if let Some(end) = filters.end_time {
1201 where_clauses.push(format!(
1202 "{time_expr} <= toDateTime64('{}', 3, 'UTC')",
1203 end.format("%Y-%m-%d %H:%M:%S%.3f")
1204 ));
1205 }
1206
1207 let mut sql = format!(
1208 "SELECT run_id, step_id, status, attempt, {time_expr} AS recorded_at, attributes FROM {}",
1209 self.step_table
1210 );
1211 if !where_clauses.is_empty() {
1212 sql.push_str(" WHERE ");
1213 sql.push_str(&where_clauses.join(" AND "));
1214 }
1215 sql.push_str(&format!(" ORDER BY {time_expr} "));
1216 sql.push_str(if filters.ascending { "ASC" } else { "DESC" });
1217 sql.push_str(", step_id ");
1218 sql.push_str(if filters.ascending { "ASC" } else { "DESC" });
1219 sql.push_str(&format!(" LIMIT {} OFFSET {}", limit, offset));
1220
1221 let rows = self
1222 .client
1223 .query(&sql)
1224 .fetch_all::<StepEventRow>()
1225 .await
1226 .context("failed to fetch step events from ClickHouse")?;
1227 Ok(rows)
1228 }
1229}
1230
1231fn parse_page_token(raw: &str) -> Result<u64, Status> {
1232 if raw.trim().is_empty() {
1233 return Ok(0);
1234 }
1235 raw.parse::<u64>()
1236 .map_err(|_| invalid_argument("page_token must be a positive integer"))
1237}
1238
1239fn timestamp_to_datetime(ts: &prost_types::Timestamp) -> Result<DateTime<Utc>, Status> {
1240 Utc.timestamp_opt(ts.seconds, ts.nanos as u32)
1241 .single()
1242 .ok_or_else(|| invalid_argument("timestamp out of range"))
1243}
1244
1245fn normalize_payload(value: Value) -> Value {
1246 match value {
1247 Value::Object(_) => value,
1248 Value::Null => Value::Object(Map::new()),
1249 other => {
1250 let mut map = Map::new();
1251 map.insert("value".to_string(), other);
1252 Value::Object(map)
1253 }
1254 }
1255}
1256
1257fn matches_filters(
1258 filters: &QueryFiltersOwned,
1259 payload: &Value,
1260 kind: &str,
1261 severity: EventSeverity,
1262 step_id: Option<Uuid>,
1263) -> bool {
1264 if !filters.kinds.is_empty() {
1265 let kind_value = kind.to_ascii_lowercase();
1266 if !filters.kinds.iter().any(|expected| expected == &kind_value) {
1267 return false;
1268 }
1269 }
1270
1271 if !filters.workspace_ids.is_empty() {
1272 let workspace = json_get_string(payload, &["trace", "run", "workspace_id"])
1273 .map(|value| value.to_ascii_lowercase());
1274 let Some(workspace) = workspace else {
1275 return false;
1276 };
1277 if !filters
1278 .workspace_ids
1279 .iter()
1280 .any(|expected| expected == &workspace)
1281 {
1282 return false;
1283 }
1284 }
1285
1286 if !filters.policy_effects.is_empty() {
1287 let effect = payload
1288 .get("effect")
1289 .and_then(Value::as_str)
1290 .map(|value| value.to_ascii_lowercase());
1291 if let Some(effect) = effect {
1292 if !filters
1293 .policy_effects
1294 .iter()
1295 .any(|expected| policy_effect_label(*expected) == effect)
1296 {
1297 return false;
1298 }
1299 } else {
1300 return false;
1301 }
1302 }
1303
1304 if !filters.severities.is_empty() && !filters.severities.contains(&severity) {
1305 return false;
1306 }
1307
1308 if filters.source == QueryEventSource::Step && !filters.step_ids.is_empty() {
1309 if let Some(step_id) = step_id {
1310 if !filters.step_ids.contains(&step_id) {
1311 return false;
1312 }
1313 } else {
1314 return false;
1315 }
1316 }
1317
1318 true
1319}
1320
1321fn policy_effect_label(effect: PolicyDecisionEffect) -> &'static str {
1322 match effect {
1323 PolicyDecisionEffect::Allow => "allow",
1324 PolicyDecisionEffect::Deny => "deny",
1325 PolicyDecisionEffect::Escalate => "escalate",
1326 PolicyDecisionEffect::Modify => "modify",
1327 _ => "",
1328 }
1329}
1330
1331fn json_get_string<'a>(value: &'a Value, path: &[&str]) -> Option<&'a str> {
1332 let mut current = value;
1333 for segment in path {
1334 current = current.get(*segment)?;
1335 }
1336 current.as_str()
1337}
1338
1339fn format_uuid_list(values: &[Uuid]) -> String {
1340 values
1341 .iter()
1342 .map(|value| format!("'{}'", value))
1343 .collect::<Vec<_>>()
1344 .join(", ")
1345}
1346
1347fn make_envelope(
1348 run_id: Uuid,
1349 step_id: Option<Uuid>,
1350 recorded_at: DateTime<Utc>,
1351 payload: &Value,
1352 kind: &str,
1353 severity: EventSeverity,
1354 raw_offset: u64,
1355) -> Result<RunEventEnvelope> {
1356 let trace = build_trace_context(run_id, step_id, payload);
1357 let tags = derive_tags(kind, step_id, payload, severity);
1358 let raw_struct = json_to_prost_struct(payload)
1359 .map_err(|err| anyhow!("failed to encode ClickHouse payload into Struct: {err}"))?;
1360 let offset =
1361 i64::try_from(raw_offset).map_err(|_| anyhow!("query offset exceeds i64 range"))?;
1362
1363 Ok(RunEventEnvelope {
1364 trace: Some(trace),
1365 recorded_at: Some(to_timestamp(recorded_at)),
1366 offset,
1367 severity: severity as i32,
1368 tags,
1369 raw: Some(raw_struct),
1370 body: None,
1371 })
1372}
1373
1374fn handle_ack(inflight: &mut VecDeque<i64>, last_ack: &mut i64, ack_offset: i64) {
1375 if ack_offset <= *last_ack {
1376 return;
1377 }
1378 *last_ack = ack_offset;
1379 while let Some(front) = inflight.front() {
1380 if *front <= ack_offset {
1381 inflight.pop_front();
1382 } else {
1383 break;
1384 }
1385 }
1386}
1387
1388fn passes_filters(
1389 config: &SubscriptionConfig,
1390 event: &models::OutboxEvent,
1391 severity: EventSeverity,
1392 tags: &[String],
1393) -> bool {
1394 if let Some(ref kinds) = config.kinds {
1395 if !kinds.contains(&event.kind.to_ascii_lowercase()) {
1396 return false;
1397 }
1398 }
1399
1400 if let Some(ref severities) = config.severities {
1401 if !severities.contains(&severity) {
1402 return false;
1403 }
1404 }
1405
1406 if let Some(ref step_filter) = config.step_ids {
1407 match event.step_id {
1408 Some(step_id) if step_filter.contains(&step_id) => {}
1409 _ => return false,
1410 }
1411 }
1412
1413 if let Some(ref tag_filter) = config.tags {
1414 if !tags
1415 .iter()
1416 .any(|tag| tag_filter.contains(&tag.to_ascii_lowercase()))
1417 {
1418 return false;
1419 }
1420 }
1421
1422 true
1423}
1424
1425fn derive_severity(kind: &str, payload: &Value) -> EventSeverity {
1426 let kind_lower = kind.to_ascii_lowercase();
1427 if kind_lower.contains("failed") || kind_lower.contains("error") {
1428 EventSeverity::Error
1429 } else if kind_lower.contains("denied") || kind_lower.contains("skipped") {
1430 EventSeverity::Warn
1431 } else if kind_lower.contains("debug") {
1432 EventSeverity::Debug
1433 } else if payload
1434 .get("severity")
1435 .and_then(Value::as_str)
1436 .map(|value| value.eq_ignore_ascii_case("warn"))
1437 .unwrap_or(false)
1438 {
1439 EventSeverity::Warn
1440 } else {
1441 EventSeverity::Info
1442 }
1443}
1444
1445fn severity_label(severity: EventSeverity) -> &'static str {
1446 match severity {
1447 EventSeverity::Trace => "trace",
1448 EventSeverity::Debug => "debug",
1449 EventSeverity::Info => "info",
1450 EventSeverity::Warn => "warn",
1451 EventSeverity::Error => "error",
1452 EventSeverity::Fatal => "fatal",
1453 EventSeverity::Unspecified => "unspecified",
1454 }
1455}
1456
1457fn derive_tags(
1458 kind: &str,
1459 step_id: Option<Uuid>,
1460 payload: &Value,
1461 severity: EventSeverity,
1462) -> Vec<String> {
1463 let mut tags = Vec::with_capacity(6);
1464 tags.push("tap".to_string());
1465 tags.push(format!("kind:{}", kind.to_ascii_lowercase()));
1466 tags.push(format!("severity:{}", severity_label(severity)));
1467
1468 if let Some(step_id) = step_id {
1469 tags.push("step".to_string());
1470 tags.push(format!("step_id:{step_id}"));
1471 } else {
1472 tags.push("run".to_string());
1473 }
1474
1475 if let Some(status) = payload.get("status").and_then(Value::as_str) {
1476 tags.push(format!("status:{}", status.to_ascii_lowercase()));
1477 }
1478 if let Some(effect) = payload.get("effect").and_then(Value::as_str) {
1479 tags.push(format!("effect:{}", effect.to_ascii_lowercase()));
1480 }
1481 tags
1482}
1483
1484fn build_response(
1485 config: &SubscriptionConfig,
1486 event: &models::OutboxEvent,
1487 severity: EventSeverity,
1488 tags: &[String],
1489 watermark: i64,
1490) -> Result<SubscribeRunResponse, Status> {
1491 let raw_struct = json_to_prost_struct(&event.payload).map_err(|err| {
1492 internal_error(format!(
1493 "failed to encode outbox payload {}: {err}",
1494 event.id
1495 ))
1496 })?;
1497
1498 let trace = build_trace_context(config.run_id, event.step_id, &event.payload);
1499
1500 let envelope = RunEventEnvelope {
1501 trace: Some(trace),
1502 recorded_at: Some(to_timestamp(event.created_at)),
1503 offset: event.id,
1504 severity: severity as i32,
1505 tags: tags.to_vec(),
1506 raw: Some(raw_struct),
1507 body: None,
1508 };
1509
1510 Ok(SubscribeRunResponse {
1511 offset: event.id,
1512 watermark,
1513 heartbeat: false,
1514 event: Some(envelope),
1515 })
1516}
1517
1518fn build_trace_context(run_id: Uuid, step_id: Option<Uuid>, payload: &Value) -> RunTraceContext {
1519 let mut ctx = RunTraceContext {
1520 trace_id: String::new(),
1521 span_id: String::new(),
1522 parent_span_id: String::new(),
1523 run: Some(tap::RunScope {
1524 run_id: run_id.to_string(),
1525 ..Default::default()
1526 }),
1527 step: None,
1528 tool_call: None,
1529 policy_decision: None,
1530 budget: None,
1531 seed: None,
1532 baggage: Vec::new(),
1533 };
1534
1535 if let Some(trace_value) = payload.get("trace").and_then(Value::as_object) {
1536 if let Some(trace_id) = trace_value.get("trace_id").and_then(Value::as_str) {
1537 ctx.trace_id = trace_id.to_string();
1538 }
1539 if let Some(span_id) = trace_value.get("span_id").and_then(Value::as_str) {
1540 ctx.span_id = span_id.to_string();
1541 }
1542 if let Some(parent_span_id) = trace_value.get("parent_span_id").and_then(Value::as_str) {
1543 ctx.parent_span_id = parent_span_id.to_string();
1544 }
1545 if let Some(seed) = trace_value.get("seed").and_then(Value::as_i64) {
1546 ctx.seed = Some(seed);
1547 }
1548 if let Some(run_obj) = trace_value.get("run").and_then(Value::as_object) {
1549 let run_id_value = run_obj
1550 .get("run_id")
1551 .and_then(Value::as_str)
1552 .map(str::to_owned)
1553 .unwrap_or_else(|| run_id.to_string());
1554 let mut run_scope = tap::RunScope {
1555 run_id: run_id_value,
1556 workspace_id: run_obj
1557 .get("workspace_id")
1558 .and_then(Value::as_str)
1559 .unwrap_or_default()
1560 .to_string(),
1561 app_id: run_obj
1562 .get("app_id")
1563 .and_then(Value::as_str)
1564 .unwrap_or_default()
1565 .to_string(),
1566 attempt_id: run_obj
1567 .get("attempt_id")
1568 .and_then(Value::as_str)
1569 .unwrap_or_default()
1570 .to_string(),
1571 labels: Default::default(),
1572 };
1573 if let Some(labels) = run_obj.get("labels").and_then(Value::as_object) {
1574 for (key, value) in labels {
1575 if let Some(string) = value.as_str() {
1576 run_scope.labels.insert(key.clone(), string.to_string());
1577 }
1578 }
1579 }
1580 ctx.run = Some(run_scope);
1581 }
1582 if let Some(step_obj) = trace_value.get("step").and_then(Value::as_object) {
1583 let mut scope = tap::StepScope {
1584 step_id: step_obj
1585 .get("step_id")
1586 .and_then(Value::as_str)
1587 .map(str::to_owned)
1588 .unwrap_or_else(|| step_id.map(|id| id.to_string()).unwrap_or_default()),
1589 index: step_obj
1590 .get("index")
1591 .and_then(Value::as_u64)
1592 .map(|value| value as u32)
1593 .unwrap_or_default(),
1594 attempt: step_obj
1595 .get("attempt")
1596 .and_then(Value::as_u64)
1597 .map(|value| value as u32)
1598 .unwrap_or_default(),
1599 kind: step_obj
1600 .get("kind")
1601 .and_then(Value::as_str)
1602 .unwrap_or_default()
1603 .to_string(),
1604 scheduler: step_obj
1605 .get("scheduler")
1606 .and_then(Value::as_str)
1607 .unwrap_or_default()
1608 .to_string(),
1609 };
1610 if scope.step_id.is_empty() {
1611 if let Some(id) = step_id {
1612 scope.step_id = id.to_string();
1613 }
1614 }
1615 ctx.step = Some(scope);
1616 } else if let Some(id) = step_id {
1617 ctx.step = Some(tap::StepScope {
1618 step_id: id.to_string(),
1619 ..Default::default()
1620 });
1621 }
1622 if let Some(tool_obj) = trace_value.get("tool_call").and_then(Value::as_object) {
1623 ctx.tool_call = Some(tap::ToolCallScope {
1624 tool_call_id: tool_obj
1625 .get("tool_call_id")
1626 .and_then(Value::as_str)
1627 .unwrap_or_default()
1628 .to_string(),
1629 tool_name: tool_obj
1630 .get("tool_name")
1631 .and_then(Value::as_str)
1632 .unwrap_or_default()
1633 .to_string(),
1634 tool_variant: tool_obj
1635 .get("tool_variant")
1636 .and_then(Value::as_str)
1637 .unwrap_or_default()
1638 .to_string(),
1639 provider: tool_obj
1640 .get("provider")
1641 .and_then(Value::as_str)
1642 .unwrap_or_default()
1643 .to_string(),
1644 });
1645 }
1646 if let Some(policy_obj) = trace_value
1647 .get("policy_decision")
1648 .and_then(Value::as_object)
1649 {
1650 ctx.policy_decision = Some(tap::PolicyDecisionScope {
1651 policy_decision_id: policy_obj
1652 .get("policy_decision_id")
1653 .and_then(Value::as_str)
1654 .unwrap_or_default()
1655 .to_string(),
1656 policy_pack_id: policy_obj
1657 .get("policy_pack_id")
1658 .and_then(Value::as_str)
1659 .unwrap_or_default()
1660 .to_string(),
1661 policy_name: policy_obj
1662 .get("policy_name")
1663 .and_then(Value::as_str)
1664 .unwrap_or_default()
1665 .to_string(),
1666 policy_version: policy_obj
1667 .get("policy_version")
1668 .and_then(Value::as_str)
1669 .unwrap_or_default()
1670 .to_string(),
1671 });
1672 }
1673 if let Some(budget_obj) = trace_value.get("budget").and_then(Value::as_object) {
1674 let mut budget_scope = tap::BudgetScope {
1675 budget_id: budget_obj
1676 .get("budget_id")
1677 .and_then(Value::as_str)
1678 .unwrap_or_default()
1679 .to_string(),
1680 budget_kind: budget_obj
1681 .get("budget_kind")
1682 .and_then(Value::as_str)
1683 .unwrap_or_default()
1684 .to_string(),
1685 amount_remaining: None,
1686 amount_allocated: None,
1687 };
1688 if let Some(value) = budget_obj.get("amount_remaining").and_then(Value::as_f64) {
1689 budget_scope.amount_remaining = Some(value);
1690 }
1691 if let Some(value) = budget_obj.get("amount_allocated").and_then(Value::as_f64) {
1692 budget_scope.amount_allocated = Some(value);
1693 }
1694 ctx.budget = Some(budget_scope);
1695 }
1696 if let Some(baggage_value) = trace_value.get("baggage") {
1697 ctx.baggage = parse_baggage(baggage_value);
1698 }
1699 } else if ctx.step.is_none() {
1700 if let Some(step_id) = step_id {
1701 ctx.step = Some(tap::StepScope {
1702 step_id: step_id.to_string(),
1703 ..Default::default()
1704 });
1705 }
1706 }
1707
1708 if ctx.run.is_none() {
1709 ctx.run = Some(tap::RunScope {
1710 run_id: run_id.to_string(),
1711 ..Default::default()
1712 });
1713 }
1714
1715 ctx
1716}
1717
1718fn parse_baggage(value: &Value) -> Vec<tap::BaggageItem> {
1719 if let Some(array) = value.as_array() {
1720 let mut items = Vec::with_capacity(array.len());
1721 for entry in array {
1722 if let Some(obj) = entry.as_object() {
1723 if let (Some(key), Some(value)) = (
1724 obj.get("key").and_then(Value::as_str),
1725 obj.get("value").and_then(Value::as_str),
1726 ) {
1727 items.push(tap::BaggageItem {
1728 key: key.to_string(),
1729 value: value.to_string(),
1730 });
1731 }
1732 }
1733 }
1734 return items;
1735 }
1736
1737 if let Some(header) = value.as_str() {
1738 return header
1739 .split(',')
1740 .filter_map(|entry| {
1741 let mut parts = entry.splitn(2, '=');
1742 let key = parts.next()?.trim();
1743 let remainder = parts.next()?.trim();
1744 let value = remainder
1745 .split_once(';')
1746 .map(|(val, _)| val.trim())
1747 .unwrap_or(remainder);
1748 if key.is_empty() || value.is_empty() {
1749 None
1750 } else {
1751 Some(tap::BaggageItem {
1752 key: key.to_string(),
1753 value: value.to_string(),
1754 })
1755 }
1756 })
1757 .collect();
1758 }
1759
1760 Vec::new()
1761}
1762
1763#[derive(Debug)]
1764struct SubmitRunSpec {
1765 dag_json: Value,
1766 inputs: Value,
1767 labels: HashMap<String, String>,
1768 seed: i64,
1769 traceparent: Option<String>,
1770 tracestate: Option<String>,
1771 baggage: Option<String>,
1772 debug: Option<Value>,
1773 breakpoints: Vec<String>,
1774}
1775
1776#[derive(Debug)]
1777enum SubmitRunOutcome {
1778 Created(Uuid),
1779 Reused(Uuid),
1780}
1781
1782#[derive(Debug)]
1783enum SubmitRunError {
1784 Invalid(String),
1785 Internal(String),
1786}
1787
1788impl std::fmt::Display for SubmitRunError {
1789 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1790 match self {
1791 SubmitRunError::Invalid(msg) => write!(f, "invalid run submission: {msg}"),
1792 SubmitRunError::Internal(msg) => write!(f, "internal run submission error: {msg}"),
1793 }
1794 }
1795}
1796
1797impl SubmitRunError {
1798 fn invalid(message: impl Into<String>) -> Self {
1799 Self::Invalid(message.into())
1800 }
1801
1802 fn internal(message: impl Into<String>) -> Self {
1803 Self::Internal(message.into())
1804 }
1805}
1806
1807fn format_schema_validation_error(err: SchemaValidationError) -> String {
1808 if err.details.is_empty() {
1809 err.schema.to_string()
1810 } else {
1811 err.details
1812 .iter()
1813 .map(|detail| detail.to_string())
1814 .collect::<Vec<_>>()
1815 .join("; ")
1816 }
1817}
1818
1819fn normalize_idempotency_key(raw: Option<&str>) -> Option<String> {
1820 raw.and_then(|value| {
1821 let trimmed = value.trim();
1822 if trimmed.is_empty() {
1823 None
1824 } else {
1825 Some(trimmed.to_owned())
1826 }
1827 })
1828}
1829
1830fn resolve_breakpoint(
1831 spec: &str,
1832 slug_map: &HashMap<String, Uuid>,
1833 step_ids: &HashSet<Uuid>,
1834) -> Result<Uuid, SubmitRunError> {
1835 let trimmed = spec.trim();
1836 if trimmed.is_empty() {
1837 return Err(SubmitRunError::invalid("breakpoint spec cannot be empty"));
1838 }
1839
1840 let (kind, target) = match trimmed.split_once(':') {
1841 Some((kind, rest)) => (kind.trim(), rest.trim()),
1842 None => ("step", trimmed),
1843 };
1844
1845 if target.is_empty() {
1846 return Err(SubmitRunError::invalid(format!(
1847 "breakpoint '{}' missing target",
1848 spec
1849 )));
1850 }
1851
1852 match kind.to_ascii_lowercase().as_str() {
1853 "step" => {
1854 if let Ok(uuid) = Uuid::parse_str(target) {
1855 if !step_ids.contains(&uuid) {
1856 return Err(SubmitRunError::invalid(format!(
1857 "breakpoint references unknown step id '{}'",
1858 target
1859 )));
1860 }
1861 return Ok(uuid);
1862 }
1863
1864 let key = target.to_ascii_lowercase();
1865 if let Some(uuid) = slug_map.get(&key) {
1866 return Ok(*uuid);
1867 }
1868
1869 Err(SubmitRunError::invalid(format!(
1870 "breakpoint references unknown step slug '{}'",
1871 target
1872 )))
1873 }
1874 other => Err(SubmitRunError::invalid(format!(
1875 "unsupported breakpoint kind '{}'",
1876 other
1877 ))),
1878 }
1879}
1880
1881async fn submit_run_impl(
1882 storage: &Storage,
1883 audit: &Audit,
1884 spec: SubmitRunSpec,
1885 idempotency_key: Option<String>,
1886 actor: &str,
1887) -> Result<SubmitRunOutcome, SubmitRunError> {
1888 if let Some(ref key) = idempotency_key {
1889 if let Some(existing_run) =
1890 storage
1891 .fetch_run_by_idempotency_key(key)
1892 .await
1893 .map_err(|err| {
1894 SubmitRunError::internal(format!("failed to check idempotency key: {err}"))
1895 })?
1896 {
1897 info!(
1898 run_id = %existing_run.run_id,
1899 idempotency_key = %key,
1900 "reusing existing run for idempotent submission"
1901 );
1902 return Ok(SubmitRunOutcome::Reused(existing_run.run_id));
1903 }
1904 }
1905
1906 let SubmitRunSpec {
1907 dag_json,
1908 inputs,
1909 labels,
1910 seed,
1911 traceparent,
1912 tracestate,
1913 baggage,
1914 debug,
1915 breakpoints,
1916 } = spec;
1917
1918 if seed <= 0 {
1919 return Err(SubmitRunError::invalid(
1920 "seed must be a positive integer to guarantee deterministic replay",
1921 ));
1922 }
1923
1924 let steps_array = dag_json
1925 .get("steps")
1926 .and_then(Value::as_array)
1927 .ok_or_else(|| SubmitRunError::invalid("dag_json.steps must be an array"))?;
1928
1929 let inputs_value = match inputs {
1930 Value::Null => Value::Object(Map::new()),
1931 Value::Object(map) => Value::Object(map),
1932 _ => return Err(SubmitRunError::invalid("inputs must be a JSON object")),
1933 };
1934
1935 let labels_value = labels_map_to_value(&labels);
1936
1937 let run_spec_instance = json!({
1938 "dag_json": dag_json.clone(),
1939 "inputs": inputs_value.clone(),
1940 "labels": labels_value.clone(),
1941 "seed": seed,
1942 "breakpoints": breakpoints.clone(),
1943 });
1944
1945 if let Err(err) = validate_run_spec(&run_spec_instance) {
1946 return Err(SubmitRunError::invalid(format!(
1947 "run spec schema validation failed: {}",
1948 format_schema_validation_error(err)
1949 )));
1950 }
1951
1952 let mut debug_map = match debug {
1953 Some(Value::Object(map)) => map,
1954 Some(_) => {
1955 return Err(SubmitRunError::invalid(
1956 "debug metadata must be a JSON object",
1957 ))
1958 }
1959 None => Map::new(),
1960 };
1961
1962 let mut ctx_map = Map::new();
1963 ctx_map.insert("inputs".to_string(), inputs_value.clone());
1964 ctx_map.insert("labels".to_string(), labels_value.clone());
1965 if traceparent.is_some() || tracestate.is_some() || baggage.is_some() {
1966 let mut trace_obj = Map::new();
1967 if let Some(tp) = traceparent {
1968 trace_obj.insert("traceparent".to_string(), Value::String(tp));
1969 }
1970 if let Some(ts) = tracestate {
1971 trace_obj.insert("tracestate".to_string(), Value::String(ts));
1972 }
1973 if let Some(bg) = baggage {
1974 trace_obj.insert("baggage".to_string(), Value::String(bg));
1975 }
1976 ctx_map.insert("trace".to_string(), Value::Object(trace_obj));
1977 }
1978
1979 let submission_time = Utc::now();
1980
1981 struct PendingStep {
1982 idx: i32,
1983 step_id: Uuid,
1984 priority: i16,
1985 spec: Value,
1986 max_attempts: i32,
1987 retry_backoff_ms: i64,
1988 retry_backoff_factor: f64,
1989 deadline_at: Option<DateTime<Utc>>,
1990 estimated_tokens: i64,
1991 estimated_cost: f64,
1992 compensation_target: Option<String>,
1993 }
1994
1995 let mut pending_steps = Vec::with_capacity(steps_array.len());
1996 let mut seen_step_ids = HashSet::with_capacity(steps_array.len());
1997 let mut slug_to_step: HashMap<String, Uuid> = HashMap::new();
1998 let mut slug_entries: Vec<(String, Uuid)> = Vec::new();
1999
2000 for (idx, step_value) in steps_array.iter().enumerate() {
2001 if let Err(err) = validate_step_spec(step_value) {
2002 let detail = err
2003 .details
2004 .iter()
2005 .map(|err| err.to_string())
2006 .collect::<Vec<_>>()
2007 .join("; ");
2008 return Err(SubmitRunError::invalid(format!(
2009 "step schema validation failed: {detail}"
2010 )));
2011 }
2012
2013 let step_obj = step_value
2014 .as_object()
2015 .ok_or_else(|| SubmitRunError::invalid("each step must be a JSON object"))?;
2016
2017 let step_id_str = step_obj
2018 .get("id")
2019 .and_then(Value::as_str)
2020 .ok_or_else(|| SubmitRunError::invalid("step.id must be a UUID string"))?;
2021 let step_id = Uuid::parse_str(step_id_str)
2022 .map_err(|err| SubmitRunError::invalid(format!("invalid step.id: {err}")))?;
2023
2024 if !seen_step_ids.insert(step_id) {
2025 return Err(SubmitRunError::invalid(format!(
2026 "duplicate step.id '{}' detected in dag_json.steps",
2027 step_id
2028 )));
2029 }
2030
2031 if let Some(slug_value) = step_obj.get("slug").and_then(Value::as_str) {
2032 let trimmed = slug_value.trim();
2033 if trimmed.is_empty() {
2034 return Err(SubmitRunError::invalid("step.slug cannot be empty"));
2035 }
2036 let key = trimmed.to_ascii_lowercase();
2037 if slug_to_step.insert(key.clone(), step_id).is_some() {
2038 return Err(SubmitRunError::invalid(format!(
2039 "duplicate step.slug '{}' detected in dag_json.steps",
2040 trimmed
2041 )));
2042 }
2043 slug_entries.push((trimmed.to_string(), step_id));
2044 }
2045
2046 let priority = extract_priority(step_obj);
2047
2048 let execution_obj = step_obj.get("execution").and_then(Value::as_object);
2049 let max_attempts = execution_obj
2050 .and_then(|exec| exec.get("max_attempts").and_then(Value::as_i64))
2051 .and_then(|v| i32::try_from(v).ok())
2052 .filter(|v| *v >= 1)
2053 .unwrap_or(1);
2054 let retry_backoff_ms = execution_obj
2055 .and_then(|exec| exec.get("retry_backoff_ms").and_then(Value::as_i64))
2056 .map(|v| v.max(0))
2057 .unwrap_or(0);
2058 let retry_backoff_factor = execution_obj
2059 .and_then(|exec| exec.get("retry_backoff_factor").and_then(Value::as_f64))
2060 .filter(|v| *v >= 1.0)
2061 .unwrap_or(1.0);
2062 let deadline_at = execution_obj
2063 .and_then(|exec| exec.get("deadline_ms").and_then(Value::as_i64))
2064 .filter(|ms| *ms > 0)
2065 .map(|ms| submission_time + chrono::Duration::milliseconds(ms));
2066
2067 let cost_hint =
2068 execution_obj.and_then(|exec| exec.get("cost_hint").and_then(Value::as_object));
2069 let estimated_tokens = cost_hint
2070 .and_then(|hint| hint.get("tokens").and_then(Value::as_i64))
2071 .map(|v| v.max(0))
2072 .unwrap_or(0);
2073 let estimated_cost = cost_hint
2074 .and_then(|hint| hint.get("usd").and_then(Value::as_f64))
2075 .map(|v| v.max(0.0))
2076 .unwrap_or(0.0);
2077
2078 let compensation_target = execution_obj
2079 .and_then(|exec| exec.get("compensation").and_then(Value::as_object))
2080 .and_then(|comp| comp.get("step").and_then(Value::as_str))
2081 .map(|s| s.to_string());
2082
2083 pending_steps.push(PendingStep {
2084 idx: idx as i32,
2085 step_id,
2086 priority,
2087 spec: step_value.clone(),
2088 max_attempts,
2089 retry_backoff_ms,
2090 retry_backoff_factor,
2091 deadline_at,
2092 estimated_tokens,
2093 estimated_cost,
2094 compensation_target,
2095 });
2096 }
2097 let mut incoming_counts: HashMap<Uuid, i32> =
2098 pending_steps.iter().map(|step| (step.step_id, 0)).collect();
2099
2100 let mut edges: Vec<(Uuid, Uuid)> = Vec::new();
2101 if let Some(edges_array) = dag_json.get("edges").and_then(Value::as_array) {
2102 let mut seen_edges = HashSet::with_capacity(edges_array.len());
2103 for edge_value in edges_array {
2104 let pair = edge_value.as_array().ok_or_else(|| {
2105 SubmitRunError::invalid("dag_json.edges entries must be [from, to] arrays")
2106 })?;
2107 if pair.len() != 2 {
2108 return Err(SubmitRunError::invalid(
2109 "dag_json.edges entries must contain exactly two elements",
2110 ));
2111 }
2112
2113 let from_str = pair[0]
2114 .as_str()
2115 .ok_or_else(|| SubmitRunError::invalid("edge source must be a UUID string"))?;
2116 let to_str = pair[1]
2117 .as_str()
2118 .ok_or_else(|| SubmitRunError::invalid("edge target must be a UUID string"))?;
2119
2120 let from = Uuid::parse_str(from_str).map_err(|err| {
2121 SubmitRunError::invalid(format!("invalid edge source UUID: {err}"))
2122 })?;
2123 let to = Uuid::parse_str(to_str).map_err(|err| {
2124 SubmitRunError::invalid(format!("invalid edge target UUID: {err}"))
2125 })?;
2126
2127 if from == to {
2128 return Err(SubmitRunError::invalid(
2129 "edge cannot reference the same step twice",
2130 ));
2131 }
2132 if !seen_step_ids.contains(&from) || !seen_step_ids.contains(&to) {
2133 return Err(SubmitRunError::invalid("edge references unknown step id"));
2134 }
2135 if !seen_edges.insert((from, to)) {
2136 return Err(SubmitRunError::invalid(
2137 "duplicate edge detected in dag_json.edges",
2138 ));
2139 }
2140
2141 let entry = incoming_counts.get_mut(&to).ok_or_else(|| {
2142 SubmitRunError::invalid("edge target not present in dag_json.steps")
2143 })?;
2144 let next_count = (*entry).checked_add(1).ok_or_else(|| {
2145 SubmitRunError::invalid("too many incoming edges for a single step")
2146 })?;
2147 *entry = next_count;
2148
2149 edges.push((from, to));
2150 }
2151 }
2152
2153 let mut breakpoint_payloads: Vec<Value> = Vec::new();
2154 let mut breakpoint_specs: Vec<String> = Vec::new();
2155 let mut breakpoint_ids: HashSet<Uuid> = HashSet::new();
2156 for bp in &breakpoints {
2157 let resolved = resolve_breakpoint(bp, &slug_to_step, &seen_step_ids)?;
2158 if !breakpoint_ids.insert(resolved) {
2159 return Err(SubmitRunError::invalid(format!(
2160 "duplicate breakpoint targeting step {}",
2161 resolved
2162 )));
2163 }
2164
2165 let mut record = Map::new();
2166 record.insert("spec".to_string(), Value::String(bp.clone()));
2167 record.insert("step_id".to_string(), Value::String(resolved.to_string()));
2168 breakpoint_payloads.push(Value::Object(record));
2169 breakpoint_specs.push(bp.clone());
2170 }
2171
2172 if !breakpoint_payloads.is_empty() {
2173 debug_map.insert(
2174 "breakpoints".to_string(),
2175 Value::Array(breakpoint_payloads.clone()),
2176 );
2177 let specs = breakpoint_specs
2178 .iter()
2179 .cloned()
2180 .map(Value::String)
2181 .collect::<Vec<_>>();
2182 debug_map.insert("breakpoint_specs".to_string(), Value::Array(specs));
2183 }
2184
2185 if !slug_entries.is_empty() {
2186 let mut slugs = Map::new();
2187 for (slug, step_id) in &slug_entries {
2188 slugs.insert(slug.clone(), Value::String(step_id.to_string()));
2189 }
2190 debug_map.insert("slugs".to_string(), Value::Object(slugs));
2191 }
2192
2193 debug_map
2194 .entry("active_pause".to_string())
2195 .or_insert(Value::Null);
2196
2197 if !debug_map.is_empty() {
2198 ctx_map.insert("debug".to_string(), Value::Object(debug_map.clone()));
2199 }
2200
2201 let input_ctx = Value::Object(ctx_map);
2202
2203 let run_id = Uuid::new_v4();
2204 let mut steps = Vec::with_capacity(pending_steps.len());
2205 for pending in pending_steps {
2206 let PendingStep {
2207 idx,
2208 step_id,
2209 priority,
2210 spec,
2211 max_attempts,
2212 retry_backoff_ms,
2213 retry_backoff_factor,
2214 deadline_at,
2215 estimated_tokens,
2216 estimated_cost,
2217 compensation_target,
2218 } = pending;
2219
2220 let dep_count = *incoming_counts.get(&step_id).unwrap_or(&0);
2221 let status = if dep_count == 0 {
2222 models::StepStatus::Queued
2223 } else {
2224 models::StepStatus::Blocked
2225 };
2226
2227 let compensation_step = if let Some(target) = compensation_target {
2228 let trimmed = target.trim();
2229 if trimmed.is_empty() {
2230 return Err(SubmitRunError::invalid(format!(
2231 "compensation step reference cannot be empty for step {}",
2232 step_id
2233 )));
2234 }
2235
2236 let resolved = if let Ok(uuid) = Uuid::parse_str(trimmed) {
2237 if !seen_step_ids.contains(&uuid) {
2238 return Err(SubmitRunError::invalid(format!(
2239 "compensation step '{}' not found in dag_json.steps",
2240 trimmed
2241 )));
2242 }
2243 uuid
2244 } else {
2245 let key = trimmed.to_ascii_lowercase();
2246 *slug_to_step.get(&key).ok_or_else(|| {
2247 SubmitRunError::invalid(format!(
2248 "compensation step '{}' not found in dag_json.steps",
2249 trimmed
2250 ))
2251 })?
2252 };
2253
2254 if resolved == step_id {
2255 return Err(SubmitRunError::invalid(format!(
2256 "step {} cannot reference itself as a compensation step",
2257 step_id
2258 )));
2259 }
2260
2261 Some(resolved)
2262 } else {
2263 None
2264 };
2265
2266 steps.push(models::NewStep {
2267 run_id,
2268 step_id,
2269 idx,
2270 priority,
2271 spec_json: spec,
2272 status,
2273 attempt: 0,
2274 input_snapshot: None,
2275 provider: None,
2276 provider_version: None,
2277 max_attempts,
2278 retry_backoff_ms,
2279 retry_backoff_factor,
2280 not_before: None,
2281 deadline_at,
2282 pending_dependencies: dep_count,
2283 total_dependencies: dep_count,
2284 estimated_tokens,
2285 estimated_cost,
2286 actual_tokens: None,
2287 actual_cost: None,
2288 checkpoint: None,
2289 compensation_step,
2290 compensation_scheduled: false,
2291 });
2292 }
2293
2294 let new_run = models::NewRun {
2295 run_id,
2296 status: models::RunStatus::Pending,
2297 dag_json: dag_json.clone(),
2298 input_ctx,
2299 seed,
2300 idempotency_key: idempotency_key.clone(),
2301 };
2302
2303 let stored_run_id = storage
2304 .insert_run_with_steps(new_run, &steps, &edges)
2305 .await
2306 .map_err(|err| {
2307 error!(error = ?err, "failed to persist run");
2308 SubmitRunError::internal(format!("failed to persist run: {err}"))
2309 })?;
2310
2311 info!(run_id = %stored_run_id, "accepted run submission");
2312
2313 let tenant_id = labels
2314 .get("tenant_id")
2315 .or_else(|| labels.get("tenant"))
2316 .and_then(|value| Uuid::parse_str(value).ok());
2317 let audit_resource = json!({
2318 "run_id": stored_run_id.to_string(),
2319 "idempotency_key": idempotency_key,
2320 "labels": labels,
2321 });
2322
2323 if let Err(err) = audit
2324 .append(actor, tenant_id, "run.submit", &audit_resource)
2325 .await
2326 {
2327 warn!(
2328 error = %err,
2329 run_id = %stored_run_id,
2330 "failed to append audit log for run submission"
2331 );
2332 }
2333
2334 Ok(SubmitRunOutcome::Created(stored_run_id))
2335}
2336
2337#[tonic::async_trait]
2338impl RuntimeService for RuntimeApiService {
2339 async fn submit_run(
2340 &self,
2341 request: Request<runtime::SubmitRunRequest>,
2342 ) -> Result<Response<runtime::SubmitRunResponse>, Status> {
2343 let metadata = request.metadata().clone();
2344 let role = self.require_role(&metadata, Role::Writer).await?;
2345 let req = request.into_inner();
2346 let runtime::SubmitRunRequest {
2347 spec,
2348 idempotency_key: raw_idempotency_key,
2349 } = req;
2350
2351 let idempotency_key = normalize_idempotency_key(Some(raw_idempotency_key.as_str()));
2352
2353 let spec = spec.ok_or_else(|| invalid_argument("spec is required"))?;
2354
2355 let dag_json = parse_dag_json(&spec.dag_json)?;
2356 let inputs_value = spec
2357 .inputs
2358 .as_ref()
2359 .map(prost_struct_to_json)
2360 .unwrap_or_else(|| Value::Object(Map::new()));
2361
2362 let traceparent = metadata
2363 .get("traceparent")
2364 .and_then(|value| value.to_str().ok())
2365 .map(|s| s.to_string());
2366 let tracestate = metadata
2367 .get("tracestate")
2368 .and_then(|value| value.to_str().ok())
2369 .map(|s| s.to_string());
2370 let baggage = metadata
2371 .get("baggage")
2372 .and_then(|value| value.to_str().ok())
2373 .map(|s| s.to_string());
2374
2375 let submission = SubmitRunSpec {
2376 dag_json,
2377 inputs: inputs_value,
2378 labels: spec.labels.clone(),
2379 seed: spec.seed,
2380 traceparent,
2381 tracestate,
2382 baggage,
2383 debug: None,
2384 breakpoints: spec.breakpoints.clone(),
2385 };
2386
2387 let actor = Self::actor_from_metadata(&metadata, role);
2388
2389 let outcome = submit_run_impl(
2390 self.storage.as_ref(),
2391 self.audit.as_ref(),
2392 submission,
2393 idempotency_key,
2394 &actor,
2395 )
2396 .await
2397 .map_err(|err| match err {
2398 SubmitRunError::Invalid(message) => invalid_argument(message),
2399 SubmitRunError::Internal(message) => internal_error(message),
2400 })?;
2401
2402 let run_id = match outcome {
2403 SubmitRunOutcome::Created(run_id) | SubmitRunOutcome::Reused(run_id) => run_id,
2404 };
2405
2406 Ok(Response::new(runtime::SubmitRunResponse {
2407 run_id: run_id.to_string(),
2408 }))
2409 }
2410
2411 async fn get_run(
2412 &self,
2413 request: Request<runtime::GetRunRequest>,
2414 ) -> Result<Response<runtime::GetRunResponse>, Status> {
2415 let metadata = request.metadata().clone();
2416 self.require_role(&metadata, Role::Reader).await?;
2417 let req = request.into_inner();
2418 let run_id = Uuid::parse_str(&req.run_id)
2419 .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2420
2421 let run = self
2422 .storage
2423 .fetch_run(run_id)
2424 .await
2425 .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
2426 .ok_or_else(|| Status::not_found("run not found"))?;
2427
2428 let steps = self
2429 .storage
2430 .fetch_steps_for_run(run_id)
2431 .await
2432 .map_err(|err| internal_error(format!("failed to fetch steps: {err}")))?;
2433
2434 let run_detail = build_run_detail(run, steps)?;
2435
2436 Ok(Response::new(runtime::GetRunResponse {
2437 run: Some(run_detail),
2438 }))
2439 }
2440
2441 type TapRunStream = Pin<Box<dyn Stream<Item = Result<runtime::RunEvent, Status>> + Send>>;
2442
2443 async fn tap_run(
2444 &self,
2445 request: Request<runtime::TapRunRequest>,
2446 ) -> Result<Response<Self::TapRunStream>, Status> {
2447 let metadata = request.metadata().clone();
2448 self.require_role(&metadata, Role::Reader).await?;
2449 let req = request.into_inner();
2450 let run_uuid = Uuid::parse_str(&req.run_id)
2451 .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2452
2453 if self
2455 .storage
2456 .fetch_run(run_uuid)
2457 .await
2458 .map_err(|err| internal_error(format!("failed to verify run: {err}")))?
2459 .is_none()
2460 {
2461 return Err(Status::not_found("run not found"));
2462 }
2463
2464 let storage = Arc::clone(&self.storage);
2465 let (tx, rx) = mpsc::channel::<Result<runtime::RunEvent, Status>>(32);
2466
2467 tokio::spawn(async move {
2468 let mut last_id: i64 = 0;
2469
2470 loop {
2471 if tx.is_closed() {
2472 break;
2473 }
2474
2475 match storage
2476 .fetch_outbox_for_run_since(run_uuid, last_id, 256)
2477 .await
2478 {
2479 Ok(events) => {
2480 if events.is_empty() {
2481 sleep(Duration::from_millis(500)).await;
2482 continue;
2483 }
2484
2485 for event in events {
2486 last_id = event.id;
2487 match run_event_from_outbox(event) {
2488 Ok(run_event) => {
2489 if tx.send(Ok(run_event)).await.is_err() {
2490 return;
2491 }
2492 }
2493 Err(status) => {
2494 let _ = tx.send(Err(status)).await;
2495 return;
2496 }
2497 }
2498 }
2499 }
2500 Err(err) => {
2501 warn!(run_id = %run_uuid, error = %err, "tap stream failed to read outbox events");
2502 let status = internal_error(format!("failed to fetch run events: {err}"));
2503 let _ = tx.send(Err(status)).await;
2504 break;
2505 }
2506 }
2507 }
2508 });
2509
2510 let stream = ReceiverStream::new(rx);
2511 Ok(Response::new(Box::pin(stream) as Self::TapRunStream))
2512 }
2513
2514 async fn replay_run(
2515 &self,
2516 request: Request<runtime::ReplayRunRequest>,
2517 ) -> Result<Response<runtime::ReplayRunResponse>, Status> {
2518 let metadata = request.metadata().clone();
2519 self.require_role(&metadata, Role::Writer).await?;
2520 let req = request.into_inner();
2521 let run_uuid = Uuid::parse_str(&req.run_id)
2522 .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2523
2524 if self
2525 .storage
2526 .fetch_run(run_uuid)
2527 .await
2528 .map_err(|err| internal_error(format!("failed to load run: {err}")))?
2529 .is_none()
2530 {
2531 return Err(Status::not_found("run not found"));
2532 }
2533
2534 let strategy = match runtime::ReplayMode::try_from(req.mode).ok() {
2535 Some(runtime::ReplayMode::Unspecified)
2536 | Some(runtime::ReplayMode::Full)
2537 | Some(runtime::ReplayMode::Diff) => runtime::ReplayStrategy::Mocked,
2538 Some(runtime::ReplayMode::StepOnly) => {
2539 return Err(Status::unimplemented(
2540 "Replay mode STEP_ONLY is not yet supported",
2541 ));
2542 }
2543 None => {
2544 return Err(invalid_argument("unrecognized replay mode"));
2545 }
2546 };
2547
2548 let steps = self
2549 .storage
2550 .fetch_steps_for_run(run_uuid)
2551 .await
2552 .map_err(|err| internal_error(format!("failed to load steps for replay: {err}")))?;
2553 let attempts = self
2554 .storage
2555 .fetch_step_attempts_for_run(run_uuid)
2556 .await
2557 .map_err(|err| {
2558 internal_error(format!("failed to load step attempts for replay: {err}"))
2559 })?;
2560
2561 let mut attempts_by_step: HashMap<Uuid, Vec<models::StepAttempt>> = HashMap::new();
2562 for attempt in attempts {
2563 attempts_by_step
2564 .entry(attempt.step_id)
2565 .or_default()
2566 .push(attempt);
2567 }
2568 for entries in attempts_by_step.values_mut() {
2569 entries.sort_by_key(|attempt| attempt.attempt);
2570 }
2571
2572 let replay_mode =
2573 runtime::ReplayMode::try_from(req.mode).unwrap_or(runtime::ReplayMode::Full);
2574 let diff_mode = matches!(replay_mode, runtime::ReplayMode::Diff);
2575
2576 let mut proto_steps = Vec::with_capacity(steps.len());
2577 let mut all_within = true;
2578
2579 for step in steps {
2580 let attempts_for_step = attempts_by_step.get(&step.step_id);
2581 let latest_attempt = attempts_for_step.and_then(|entries| entries.last());
2582
2583 let recorded_snapshot = latest_attempt
2584 .and_then(|attempt| attempt.output_snapshot.clone())
2585 .or_else(|| step.output_snapshot.clone())
2586 .ok_or_else(|| {
2587 Status::failed_precondition(format!(
2588 "step {} missing output_snapshot; cannot replay",
2589 step.step_id
2590 ))
2591 })?;
2592
2593 let baseline_attempt = if diff_mode {
2594 attempts_for_step.and_then(|entries| {
2595 if entries.len() >= 2 {
2596 entries.get(entries.len() - 2)
2597 } else {
2598 None
2599 }
2600 })
2601 } else {
2602 latest_attempt
2603 };
2604
2605 let baseline_output = baseline_attempt
2606 .and_then(|attempt| attempt.output_snapshot.clone())
2607 .unwrap_or_else(|| recorded_snapshot.clone());
2608
2609 let current_output = if diff_mode {
2610 recorded_snapshot.clone()
2611 } else {
2612 step.output_json
2613 .clone()
2614 .unwrap_or_else(|| recorded_snapshot.clone())
2615 };
2616
2617 let drift_result = compute_drift(
2618 &baseline_output,
2619 ¤t_output,
2620 DEFAULT_TOKEN_TOLERANCE,
2621 DEFAULT_COST_TOLERANCE,
2622 );
2623 if !drift_result.within_tolerance {
2624 all_within = false;
2625 }
2626
2627 let snapshot_value = json_to_prost_value(&recorded_snapshot)
2628 .map_err(|err| internal_error(format!("failed to encode snapshot: {err}")))?;
2629
2630 let recorded_meta = wrap_attempt_output(baseline_attempt, &baseline_output);
2631 let current_meta = wrap_attempt_output(latest_attempt, ¤t_output);
2632
2633 let drift_proto = runtime::ReplayStepDrift {
2634 recorded: Some(json_to_prost_value(&recorded_meta).map_err(|err| {
2635 internal_error(format!("failed to encode recorded drift snapshot: {err}"))
2636 })?),
2637 current: Some(json_to_prost_value(¤t_meta).map_err(|err| {
2638 internal_error(format!("failed to encode current drift snapshot: {err}"))
2639 })?),
2640 token_delta: drift_result.token_delta,
2641 cost_delta: drift_result.cost_delta,
2642 within_tolerance: drift_result.within_tolerance,
2643 };
2644
2645 proto_steps.push(runtime::ReplayStep {
2646 step_id: step.step_id.to_string(),
2647 snapshot: Some(snapshot_value),
2648 drift: Some(drift_proto),
2649 });
2650 }
2651
2652 let response = runtime::ReplayRunResponse {
2653 run_id: req.run_id,
2654 all_within_tolerance: all_within,
2655 strategy: strategy as i32,
2656 steps: proto_steps,
2657 };
2658
2659 Ok(Response::new(response))
2660 }
2661
2662 async fn update_breakpoints(
2663 &self,
2664 request: Request<runtime::UpdateBreakpointsRequest>,
2665 ) -> Result<Response<runtime::UpdateBreakpointsResponse>, Status> {
2666 let metadata = request.metadata().clone();
2667 let role = self.require_role(&metadata, Role::Writer).await?;
2668 let actor = Self::actor_from_metadata(&metadata, role);
2669 let req = request.into_inner();
2670
2671 let run_uuid = Uuid::parse_str(&req.run_id)
2672 .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2673
2674 if !req.clear_all && req.add.is_empty() && req.remove.is_empty() {
2675 return Err(invalid_argument(
2676 "update_breakpoints requires add/remove entries or clear_all",
2677 ));
2678 }
2679
2680 let run = self
2681 .storage
2682 .fetch_run(run_uuid)
2683 .await
2684 .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
2685 .ok_or_else(|| Status::not_found("run not found"))?;
2686
2687 let step_ids = collect_step_ids(&run);
2688
2689 let ctx_value = run.input_ctx.clone();
2690 let mut ctx_map = match ctx_value {
2691 Value::Object(map) => map,
2692 Value::Null => Map::new(),
2693 other => {
2694 return Err(internal_error(format!(
2695 "run input context must be an object, found {other}"
2696 )))
2697 }
2698 };
2699
2700 let debug_entry = ctx_map
2701 .entry("debug".to_string())
2702 .or_insert_with(|| Value::Object(Map::new()));
2703 let debug_map = debug_entry.as_object_mut().ok_or_else(|| {
2704 internal_error("run debug context is not an object; unable to update breakpoints")
2705 })?;
2706
2707 let mut breakpoints = load_debug_breakpoints(debug_map);
2708 let existing_slug_entries = load_slug_map(debug_map);
2709
2710 let mut slug_lookup: HashMap<String, Uuid> = existing_slug_entries
2711 .iter()
2712 .map(|(slug, id)| (slug.to_ascii_lowercase(), *id))
2713 .collect();
2714 if slug_lookup.is_empty() {
2715 for (slug, id) in collect_slug_entries(&run) {
2716 slug_lookup.insert(slug.to_ascii_lowercase(), id);
2717 }
2718 }
2719
2720 let slug_entries_store: Vec<(String, Uuid)> = if existing_slug_entries.is_empty() {
2721 collect_slug_entries(&run)
2722 } else {
2723 existing_slug_entries
2724 .iter()
2725 .map(|(slug, id)| (slug.clone(), *id))
2726 .collect()
2727 };
2728
2729 if req.clear_all {
2730 breakpoints.clear();
2731 }
2732
2733 let removal_specs: HashSet<String> = req
2734 .remove
2735 .iter()
2736 .map(|s| s.trim().to_string())
2737 .filter(|s| !s.is_empty())
2738 .collect();
2739
2740 let mut removal_ids: HashSet<Uuid> = HashSet::new();
2741 for spec in &req.remove {
2742 if let Ok(step_id) = resolve_breakpoint(spec, &slug_lookup, &step_ids) {
2743 removal_ids.insert(step_id);
2744 }
2745 }
2746
2747 if !removal_specs.is_empty() || !removal_ids.is_empty() {
2748 breakpoints.retain(|(spec, step_id)| {
2749 let spec_trim = spec.trim();
2750 let matches_spec = removal_specs.contains(spec_trim);
2751 let matches_id = removal_ids.contains(step_id);
2752 !(matches_spec || matches_id)
2753 });
2754 }
2755
2756 for spec in &req.add {
2757 let step_id = resolve_breakpoint(spec, &slug_lookup, &step_ids)
2758 .map_err(|err| invalid_argument(err.to_string()))?;
2759 breakpoints.retain(|(_, existing)| existing != &step_id);
2760 breakpoints.push((spec.trim().to_string(), step_id));
2761 }
2762
2763 breakpoints.sort_by(|a, b| a.0.cmp(&b.0));
2764 store_debug_breakpoints(debug_map, &breakpoints);
2765 if !slug_entries_store.is_empty() {
2766 store_slug_map(debug_map, &slug_entries_store);
2767 }
2768
2769 let mut tx = self
2770 .storage
2771 .pool()
2772 .begin()
2773 .await
2774 .map_err(|err| internal_error(format!("failed to begin transaction: {err}")))?;
2775
2776 self.storage
2777 .update_run_input_ctx_tx(&mut tx, run_uuid, Value::Object(ctx_map.clone()))
2778 .await
2779 .map_err(|err| {
2780 internal_error(format!("failed to persist breakpoint changes: {err}"))
2781 })?;
2782
2783 let payload_breakpoints = breakpoints
2784 .iter()
2785 .map(|(spec, step_id)| {
2786 json!({
2787 "spec": spec,
2788 "step_id": step_id.to_string(),
2789 })
2790 })
2791 .collect::<Vec<_>>();
2792
2793 self.storage
2794 .enqueue_outbox_event(
2795 &mut tx,
2796 NewOutboxEvent {
2797 run_id: run_uuid,
2798 step_id: None,
2799 kind: "run_breakpoints_updated".to_string(),
2800 payload: json!({
2801 "run_id": run_uuid.to_string(),
2802 "breakpoints": payload_breakpoints,
2803 "added": req.add,
2804 "removed": req.remove,
2805 "cleared": req.clear_all,
2806 }),
2807 },
2808 )
2809 .await
2810 .map_err(|err| internal_error(format!("failed to enqueue outbox event: {err}")))?;
2811
2812 tx.commit()
2813 .await
2814 .map_err(|err| internal_error(format!("failed to commit breakpoint update: {err}")))?;
2815
2816 if let Err(err) = self
2817 .audit
2818 .append(
2819 &actor,
2820 self.storage
2821 .fetch_run_tenant(run_uuid)
2822 .await
2823 .ok()
2824 .flatten()
2825 .and_then(|tenant| Uuid::parse_str(&tenant).ok()),
2826 "debug.breakpoints.update",
2827 &json!({
2828 "run_id": run_uuid.to_string(),
2829 "added": req.add,
2830 "removed": req.remove,
2831 "cleared": req.clear_all,
2832 }),
2833 )
2834 .await
2835 {
2836 warn!(error = %err, run = %run_uuid, "failed to append audit record for breakpoint update");
2837 }
2838
2839 Ok(Response::new(runtime::UpdateBreakpointsResponse {
2840 breakpoints: breakpoints.into_iter().map(|(spec, _)| spec).collect(),
2841 }))
2842 }
2843
2844 async fn resume_run(
2845 &self,
2846 request: Request<runtime::ResumeRunRequest>,
2847 ) -> Result<Response<runtime::ResumeRunResponse>, Status> {
2848 let metadata = request.metadata().clone();
2849 let role = self.require_role(&metadata, Role::Writer).await?;
2850 let actor = Self::actor_from_metadata(&metadata, role);
2851 let req = request.into_inner();
2852
2853 let run_uuid = Uuid::parse_str(&req.run_id)
2854 .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2855
2856 let run = self
2857 .storage
2858 .fetch_run(run_uuid)
2859 .await
2860 .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
2861 .ok_or_else(|| Status::not_found("run not found"))?;
2862
2863 if run.status != models::RunStatus::PausedAtStep {
2864 return Err(Status::failed_precondition(
2865 "run is not paused at a breakpoint",
2866 ));
2867 }
2868
2869 let ctx_value = run.input_ctx.clone();
2870 let mut ctx_map = match ctx_value {
2871 Value::Object(map) => map,
2872 Value::Null => Map::new(),
2873 other => {
2874 return Err(internal_error(format!(
2875 "run input context must be an object, found {other}"
2876 )))
2877 }
2878 };
2879
2880 let debug_entry = ctx_map
2881 .get_mut("debug")
2882 .and_then(Value::as_object_mut)
2883 .ok_or_else(|| internal_error("run debug context missing; cannot resume paused run"))?;
2884
2885 let paused_step = extract_active_pause_step_id(debug_entry).ok_or_else(|| {
2886 Status::failed_precondition("run is paused but no active step recorded")
2887 })?;
2888
2889 debug_entry.insert("active_pause".to_string(), Value::Null);
2890
2891 let mut tx = self
2892 .storage
2893 .pool()
2894 .begin()
2895 .await
2896 .map_err(|err| internal_error(format!("failed to begin transaction: {err}")))?;
2897
2898 self.storage
2899 .update_run_input_ctx_tx(&mut tx, run_uuid, Value::Object(ctx_map.clone()))
2900 .await
2901 .map_err(|err| internal_error(format!("failed to persist pause state: {err}")))?;
2902
2903 self.storage
2904 .set_run_status_tx(&mut tx, run_uuid, models::RunStatus::Running)
2905 .await
2906 .map_err(|err| internal_error(format!("failed to update run status: {err}")))?;
2907
2908 self.storage
2909 .enqueue_outbox_event(
2910 &mut tx,
2911 NewOutboxEvent {
2912 run_id: run_uuid,
2913 step_id: Some(paused_step),
2914 kind: "run_resumed".to_string(),
2915 payload: json!({
2916 "run_id": run_uuid.to_string(),
2917 "step_id": paused_step.to_string(),
2918 }),
2919 },
2920 )
2921 .await
2922 .map_err(|err| internal_error(format!("failed to enqueue resume event: {err}")))?;
2923
2924 tx.commit()
2925 .await
2926 .map_err(|err| internal_error(format!("failed to commit resume transaction: {err}")))?;
2927
2928 if let Err(err) = self
2929 .audit
2930 .append(
2931 &actor,
2932 self.storage
2933 .fetch_run_tenant(run_uuid)
2934 .await
2935 .ok()
2936 .flatten()
2937 .and_then(|tenant| Uuid::parse_str(&tenant).ok()),
2938 "debug.breakpoints.resume",
2939 &json!({
2940 "run_id": run_uuid.to_string(),
2941 "step_id": paused_step.to_string(),
2942 }),
2943 )
2944 .await
2945 {
2946 warn!(error = %err, run = %run_uuid, "failed to append audit record for resume");
2947 }
2948
2949 Ok(Response::new(runtime::ResumeRunResponse {
2950 status: runtime::RunStatus::Running as i32,
2951 resumed_step_id: paused_step.to_string(),
2952 }))
2953 }
2954
2955 async fn patch_step(
2956 &self,
2957 request: Request<runtime::PatchStepRequest>,
2958 ) -> Result<Response<runtime::PatchStepResponse>, Status> {
2959 let metadata = request.metadata().clone();
2960 let role = self.require_role(&metadata, Role::Writer).await?;
2961 let actor = Self::actor_from_metadata(&metadata, role);
2962 let req = request.into_inner();
2963
2964 let run_uuid = Uuid::parse_str(&req.run_id)
2965 .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
2966 let step_uuid = Uuid::parse_str(&req.step_id)
2967 .map_err(|err| invalid_argument(format!("invalid step_id: {err}")))?;
2968
2969 let run = self
2970 .storage
2971 .fetch_run(run_uuid)
2972 .await
2973 .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
2974 .ok_or_else(|| Status::not_found("run not found"))?;
2975
2976 if run.status != models::RunStatus::PausedAtStep {
2977 return Err(Status::failed_precondition(
2978 "run must be paused before patching step",
2979 ));
2980 }
2981
2982 let ctx_value = run.input_ctx.clone();
2983 let paused_step = match ctx_value {
2984 Value::Object(ref map) => map
2985 .get("debug")
2986 .and_then(Value::as_object)
2987 .and_then(extract_active_pause_step_id),
2988 _ => None,
2989 };
2990
2991 if let Some(active_step) = paused_step {
2992 if active_step != step_uuid {
2993 return Err(Status::failed_precondition(
2994 "only the paused step can be patched",
2995 ));
2996 }
2997 }
2998
2999 let inputs_patch_value = req.inputs_patch.as_ref().map(prost_struct_to_json);
3000 let env_override = req.env.as_ref().map(prost_struct_to_json);
3001
3002 let model_override = req.model.clone();
3003
3004 if inputs_patch_value.is_none() && env_override.is_none() && model_override.is_none() {
3005 return Err(invalid_argument(
3006 "patch_step requires inputs_patch, env, or model to be set",
3007 ));
3008 }
3009
3010 let steps = self
3011 .storage
3012 .fetch_steps_for_run(run_uuid)
3013 .await
3014 .map_err(|err| internal_error(format!("failed to fetch steps: {err}")))?;
3015
3016 let step = steps
3017 .into_iter()
3018 .find(|step| step.step_id == step_uuid)
3019 .ok_or_else(|| Status::not_found("step not found"))?;
3020
3021 let mut spec_map = match step.spec_json.clone() {
3022 Value::Object(map) => map,
3023 _ => {
3024 return Err(internal_error(
3025 "step spec must be a JSON object to apply patches",
3026 ))
3027 }
3028 };
3029
3030 let inputs_entry = spec_map
3031 .entry("inputs".to_string())
3032 .or_insert_with(|| Value::Object(Map::new()));
3033 let inputs_map = inputs_entry
3034 .as_object_mut()
3035 .ok_or_else(|| internal_error("step inputs is not an object; cannot apply patch"))?;
3036
3037 if let Some(patch_value) = inputs_patch_value.as_ref() {
3038 let patch_obj = patch_value
3039 .as_object()
3040 .ok_or_else(|| invalid_argument("inputs_patch must be a JSON object"))?;
3041 merge_json_object(inputs_map, patch_obj);
3042 }
3043
3044 if let Some(env_value) = env_override.as_ref() {
3045 if !env_value.is_object() {
3046 return Err(invalid_argument("env override must be a JSON object"));
3047 }
3048 inputs_map.insert("env".to_string(), env_value.clone());
3049 }
3050
3051 if let Some(model) = model_override.as_ref() {
3052 inputs_map.insert("model".to_string(), Value::String(model.clone()));
3053 }
3054
3055 let updated_spec = Value::Object(spec_map.clone());
3056
3057 let mut tx = self
3058 .storage
3059 .pool()
3060 .begin()
3061 .await
3062 .map_err(|err| internal_error(format!("failed to begin transaction: {err}")))?;
3063
3064 self.storage
3065 .update_step_spec_tx(&mut tx, run_uuid, step_uuid, updated_spec.clone())
3066 .await
3067 .map_err(|err| internal_error(format!("failed to persist step patch: {err}")))?;
3068
3069 self.storage
3070 .enqueue_outbox_event(
3071 &mut tx,
3072 NewOutboxEvent {
3073 run_id: run_uuid,
3074 step_id: Some(step_uuid),
3075 kind: "step_patched".to_string(),
3076 payload: json!({
3077 "run_id": run_uuid.to_string(),
3078 "step_id": step_uuid.to_string(),
3079 "inputs_patch": inputs_patch_value,
3080 "env_override": env_override,
3081 "model_override": model_override,
3082 "note": req.note,
3083 }),
3084 },
3085 )
3086 .await
3087 .map_err(|err| internal_error(format!("failed to enqueue patch event: {err}")))?;
3088
3089 tx.commit()
3090 .await
3091 .map_err(|err| internal_error(format!("failed to commit patch transaction: {err}")))?;
3092
3093 if let Err(err) = self
3094 .audit
3095 .append(
3096 &actor,
3097 self.storage
3098 .fetch_run_tenant(run_uuid)
3099 .await
3100 .ok()
3101 .flatten()
3102 .and_then(|tenant| Uuid::parse_str(&tenant).ok()),
3103 "debug.step.patch",
3104 &json!({
3105 "run_id": run_uuid.to_string(),
3106 "step_id": step_uuid.to_string(),
3107 "note": req.note,
3108 }),
3109 )
3110 .await
3111 {
3112 warn!(error = %err, run = %run_uuid, step = %step_uuid, "failed to append audit record for step patch");
3113 }
3114
3115 Ok(Response::new(runtime::PatchStepResponse {
3116 step_id: step_uuid.to_string(),
3117 }))
3118 }
3119
3120 async fn create_branch_run(
3121 &self,
3122 request: Request<runtime::CreateBranchRunRequest>,
3123 ) -> Result<Response<runtime::CreateBranchRunResponse>, Status> {
3124 let metadata = request.metadata().clone();
3125 let role = self.require_role(&metadata, Role::Writer).await?;
3126 let actor = Self::actor_from_metadata(&metadata, role);
3127 let req = request.into_inner();
3128
3129 let run_uuid = Uuid::parse_str(&req.run_id)
3130 .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
3131
3132 let from_step_spec = req.from_step.trim().to_string();
3133 if from_step_spec.is_empty() {
3134 return Err(invalid_argument("from_step is required"));
3135 }
3136
3137 let seed_mode = runtime::BranchSeedMode::try_from(req.seed_mode)
3138 .unwrap_or(runtime::BranchSeedMode::Same);
3139
3140 let run = self
3141 .storage
3142 .fetch_run(run_uuid)
3143 .await
3144 .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
3145 .ok_or_else(|| Status::not_found("run not found"))?;
3146
3147 let (inputs_value, mut labels_map) = unpack_input_ctx(&run.input_ctx);
3148
3149 let debug_entry = run
3150 .input_ctx
3151 .get("debug")
3152 .and_then(Value::as_object)
3153 .cloned()
3154 .unwrap_or_default();
3155 let slug_lookup = load_slug_map(&debug_entry);
3156 let step_ids = collect_step_ids(&run);
3157 let target_step = resolve_breakpoint(&from_step_spec, &slug_lookup, &step_ids)
3158 .map_err(|err| invalid_argument(err.to_string()))?;
3159
3160 let branch_dag = build_branch_dag(&run.dag_json, target_step)
3161 .map_err(|err| invalid_argument(err.to_string()))?;
3162
3163 let mut inputs_map = match inputs_value {
3164 Value::Object(map) => map,
3165 Value::Null => Map::new(),
3166 _ => Map::new(),
3167 };
3168
3169 let mut inputs_patch_debug: Option<Value> = None;
3170 if let Some(patch) = req.inputs_patch.as_ref() {
3171 let patch_value = prost_struct_to_json(patch);
3172 let patch_obj = patch_value
3173 .as_object()
3174 .ok_or_else(|| invalid_argument("inputs_patch must be a JSON object"))?;
3175 merge_json_object(&mut inputs_map, patch_obj);
3176 inputs_patch_debug = Some(Value::Object(patch_obj.clone()));
3177 }
3178
3179 let mut labels_patch_debug: Option<Value> = None;
3180 if let Some(labels_patch) = req.labels_patch.as_ref() {
3181 let patch_value = prost_struct_to_json(labels_patch);
3182 let patch_obj = patch_value
3183 .as_object()
3184 .ok_or_else(|| invalid_argument("labels_patch must be a JSON object"))?;
3185 for (key, value) in patch_obj {
3186 let rendered = value
3187 .as_str()
3188 .map(|s| s.to_string())
3189 .unwrap_or_else(|| value.to_string());
3190 labels_map.insert(key.clone(), rendered);
3191 }
3192 labels_patch_debug = Some(Value::Object(patch_obj.clone()));
3193 }
3194
3195 labels_map.insert("branch_parent".to_string(), run_uuid.to_string());
3196 labels_map.insert("branch_from_step".to_string(), target_step.to_string());
3197
3198 let branch_seed = match seed_mode {
3199 runtime::BranchSeedMode::Same | runtime::BranchSeedMode::Unspecified => run.seed,
3200 runtime::BranchSeedMode::New => Utc::now().timestamp_nanos_opt().unwrap_or(run.seed),
3201 runtime::BranchSeedMode::Custom => req
3202 .seed
3203 .as_ref()
3204 .copied()
3205 .ok_or_else(|| invalid_argument("seed value required for custom mode"))?,
3206 };
3207
3208 let prompt_packs = collect_prompt_packs_json(&run.dag_json);
3209
3210 let mut branch_meta = Map::new();
3211 branch_meta.insert(
3212 "parent_run_id".to_string(),
3213 Value::String(run_uuid.to_string()),
3214 );
3215 branch_meta.insert(
3216 "parent_step_id".to_string(),
3217 Value::String(target_step.to_string()),
3218 );
3219 branch_meta.insert(
3220 "from_step_spec".to_string(),
3221 Value::String(from_step_spec.clone()),
3222 );
3223 branch_meta.insert(
3224 "parent_seed".to_string(),
3225 Value::Number(Number::from(run.seed)),
3226 );
3227 branch_meta.insert(
3228 "branch_seed".to_string(),
3229 Value::Number(Number::from(branch_seed)),
3230 );
3231 let seed_mode_str = match seed_mode {
3232 runtime::BranchSeedMode::Same | runtime::BranchSeedMode::Unspecified => "same",
3233 runtime::BranchSeedMode::New => "new",
3234 runtime::BranchSeedMode::Custom => "custom",
3235 };
3236 branch_meta.insert(
3237 "seed_mode".to_string(),
3238 Value::String(seed_mode_str.to_string()),
3239 );
3240 branch_meta.insert(
3241 "created_at".to_string(),
3242 Value::String(Utc::now().to_rfc3339()),
3243 );
3244 branch_meta.insert(
3245 "prompt_packs".to_string(),
3246 Value::Array(prompt_packs.into_iter().map(Value::String).collect()),
3247 );
3248 let note = req.note.trim();
3249 if !note.is_empty() {
3250 branch_meta.insert("note".to_string(), Value::String(note.to_string()));
3251 }
3252 if let Some(patch) = inputs_patch_debug {
3253 branch_meta.insert("inputs_patch".to_string(), patch);
3254 }
3255 if let Some(patch) = labels_patch_debug {
3256 branch_meta.insert("labels_patch".to_string(), patch);
3257 }
3258
3259 let mut debug_extra = Map::new();
3260 debug_extra.insert("branch".to_string(), Value::Object(branch_meta));
3261
3262 let submission = SubmitRunSpec {
3263 dag_json: branch_dag,
3264 inputs: Value::Object(inputs_map.clone()),
3265 labels: labels_map.clone(),
3266 seed: branch_seed,
3267 traceparent: None,
3268 tracestate: None,
3269 baggage: None,
3270 debug: Some(Value::Object(debug_extra.clone())),
3271 breakpoints: Vec::new(),
3272 };
3273
3274 let outcome = submit_run_impl(
3275 self.storage.as_ref(),
3276 self.audit.as_ref(),
3277 submission,
3278 None,
3279 &actor,
3280 )
3281 .await
3282 .map_err(|err| match err {
3283 SubmitRunError::Invalid(message) => invalid_argument(message),
3284 SubmitRunError::Internal(message) => internal_error(message),
3285 })?;
3286
3287 let new_run_uuid = match outcome {
3288 SubmitRunOutcome::Created(run_id) | SubmitRunOutcome::Reused(run_id) => run_id,
3289 };
3290
3291 let mut tx = self.storage.pool().begin().await.map_err(|err| {
3292 internal_error(format!("failed to begin branch event transaction: {err}"))
3293 })?;
3294
3295 self.storage
3296 .enqueue_outbox_event(
3297 &mut tx,
3298 NewOutboxEvent {
3299 run_id: run_uuid,
3300 step_id: Some(target_step),
3301 kind: "run_branch_created".to_string(),
3302 payload: json!({
3303 "run_id": run_uuid.to_string(),
3304 "child_run_id": new_run_uuid.to_string(),
3305 "from_step_spec": from_step_spec,
3306 "seed": branch_seed,
3307 }),
3308 },
3309 )
3310 .await
3311 .map_err(|err| internal_error(format!("failed to enqueue branch event: {err}")))?;
3312
3313 tx.commit().await.map_err(|err| {
3314 internal_error(format!("failed to commit branch event transaction: {err}"))
3315 })?;
3316
3317 if let Err(err) = self
3318 .audit
3319 .append(
3320 &actor,
3321 self.storage
3322 .fetch_run_tenant(run_uuid)
3323 .await
3324 .ok()
3325 .flatten()
3326 .and_then(|tenant| Uuid::parse_str(&tenant).ok()),
3327 "debug.branch.create",
3328 &json!({
3329 "run_id": run_uuid.to_string(),
3330 "child_run_id": new_run_uuid.to_string(),
3331 "step_id": target_step.to_string(),
3332 }),
3333 )
3334 .await
3335 {
3336 warn!(error = %err, run = %run_uuid, "failed to append audit record for branch");
3337 }
3338
3339 Ok(Response::new(runtime::CreateBranchRunResponse {
3340 run_id: new_run_uuid.to_string(),
3341 }))
3342 }
3343
3344 async fn get_attestations(
3345 &self,
3346 request: Request<runtime::GetAttestationsRequest>,
3347 ) -> Result<Response<runtime::GetAttestationsResponse>, Status> {
3348 let metadata = request.metadata().clone();
3349 self.require_role(&metadata, Role::Reader).await?;
3350 let req = request.into_inner();
3351 let mut seen: HashSet<Uuid> = HashSet::new();
3352 let mut results = Vec::new();
3353
3354 for raw in req.attestation_ids {
3355 let trimmed = raw.trim();
3356 if trimmed.is_empty() {
3357 continue;
3358 }
3359 let id = Uuid::parse_str(trimmed).map_err(|err| {
3360 invalid_argument(format!("invalid attestation id '{trimmed}': {err}"))
3361 })?;
3362 if !seen.insert(id) {
3363 continue;
3364 }
3365 match self.attestation_vault.fetch(id).await {
3366 Ok(Some(attestation)) => {
3367 results.push(attestation_to_proto(attestation)?);
3368 }
3369 Ok(None) => {
3370 }
3372 Err(err) => {
3373 return Err(internal_error(format!(
3374 "failed to load attestation {id}: {err}"
3375 )));
3376 }
3377 }
3378 }
3379
3380 Ok(Response::new(runtime::GetAttestationsResponse {
3381 attestations: results,
3382 }))
3383 }
3384
3385 async fn sync_eval_pack(
3386 &self,
3387 request: Request<runtime::SyncEvalPackRequest>,
3388 ) -> Result<Response<runtime::SyncEvalPackResponse>, Status> {
3389 let metadata = request.metadata().clone();
3390 self.require_role(&metadata, Role::Writer).await?;
3391
3392 let req = request.into_inner();
3393 let pack = req.pack.trim();
3394 if pack.is_empty() {
3395 return Err(invalid_argument("pack is required"));
3396 }
3397 let pack_name = pack.to_string();
3398
3399 let mut scenario_ids = Vec::with_capacity(req.scenarios.len());
3400 for scenario in req.scenarios {
3401 let slug = scenario.slug.trim();
3402 if slug.is_empty() {
3403 return Err(invalid_argument("scenario slug cannot be empty"));
3404 }
3405 let run_spec_struct = scenario
3406 .run_spec
3407 .ok_or_else(|| invalid_argument("scenario run_spec must be provided"))?;
3408 let expected_struct = scenario
3409 .expected
3410 .ok_or_else(|| invalid_argument("scenario expected payload must be provided"))?;
3411
3412 let run_spec_json = prost_struct_to_json(&run_spec_struct);
3414 let expected_json = prost_struct_to_json(&expected_struct);
3415
3416 let title = scenario.title.trim();
3417 let description = scenario.description.trim();
3418
3419 let mut spec_map = Map::new();
3420 spec_map.insert("pack".to_string(), Value::String(pack_name.clone()));
3421 spec_map.insert("slug".to_string(), Value::String(slug.to_string()));
3422 if !title.is_empty() {
3423 spec_map.insert("title".to_string(), Value::String(title.to_string()));
3424 }
3425 if !description.is_empty() {
3426 spec_map.insert(
3427 "description".to_string(),
3428 Value::String(description.to_string()),
3429 );
3430 }
3431 spec_map.insert("run_spec".to_string(), run_spec_json);
3432 spec_map.insert("expected".to_string(), expected_json);
3433 let spec_value = Value::Object(spec_map);
3434
3435 let scenario_name = format!("{pack}/{slug}");
3436 let scenario_id = self
3437 .storage
3438 .upsert_eval_scenario(
3439 &scenario_name,
3440 if description.is_empty() {
3441 None
3442 } else {
3443 Some(description)
3444 },
3445 spec_value,
3446 )
3447 .await
3448 .map_err(|err| {
3449 internal_error(format!(
3450 "failed to upsert eval scenario '{scenario_name}': {err}"
3451 ))
3452 })?;
3453 scenario_ids.push(scenario_id.to_string());
3454 }
3455
3456 Ok(Response::new(runtime::SyncEvalPackResponse {
3457 scenario_ids,
3458 }))
3459 }
3460
3461 async fn list_eval_scenarios(
3462 &self,
3463 request: Request<runtime::ListEvalScenariosRequest>,
3464 ) -> Result<Response<runtime::ListEvalScenariosResponse>, Status> {
3465 let metadata = request.metadata().clone();
3466 self.require_role(&metadata, Role::Reader).await?;
3467
3468 let scenarios = self
3469 .storage
3470 .list_eval_scenarios()
3471 .await
3472 .map_err(|err| internal_error(format!("failed to list eval scenarios: {err}")))?;
3473
3474 let mut proto_scenarios = Vec::with_capacity(scenarios.len());
3475 for scenario in scenarios {
3476 let spec = parse_eval_spec(&scenario.spec).map_err(|err| {
3477 internal_error(format!(
3478 "invalid eval scenario '{}' spec: {err}",
3479 scenario.name
3480 ))
3481 })?;
3482 let history_models = self
3483 .storage
3484 .list_eval_results(scenario.scenario_id, 20)
3485 .await
3486 .map_err(|err| internal_error(format!("failed to fetch eval results: {err}")))?;
3487 let mut history_proto = Vec::with_capacity(history_models.len());
3488 for result in history_models {
3489 history_proto.push(build_eval_result_proto(result)?);
3490 }
3491 proto_scenarios.push(build_eval_scenario_proto(&scenario, &spec, history_proto)?);
3492 }
3493
3494 Ok(Response::new(runtime::ListEvalScenariosResponse {
3495 scenarios: proto_scenarios,
3496 }))
3497 }
3498
3499 async fn record_eval_result(
3500 &self,
3501 request: Request<runtime::RecordEvalResultRequest>,
3502 ) -> Result<Response<runtime::RecordEvalResultResponse>, Status> {
3503 let metadata = request.metadata().clone();
3504 self.require_role(&metadata, Role::Writer).await?;
3505
3506 let req = request.into_inner();
3507 let pack = req.pack.trim();
3508 let slug = req.slug.trim();
3509 if pack.is_empty() {
3510 return Err(invalid_argument("pack is required"));
3511 }
3512 if slug.is_empty() {
3513 return Err(invalid_argument("slug is required"));
3514 }
3515
3516 let scenario_name = format!("{pack}/{slug}");
3517 let scenario = self
3518 .storage
3519 .fetch_eval_scenario_by_name(&scenario_name)
3520 .await
3521 .map_err(|err| internal_error(format!("failed to fetch eval scenario: {err}")))?
3522 .ok_or_else(|| {
3523 Status::not_found(format!("eval scenario '{scenario_name}' not found"))
3524 })?;
3525
3526 let spec = parse_eval_spec(&scenario.spec).map_err(|err| {
3527 internal_error(format!(
3528 "invalid eval scenario '{}' spec: {err}",
3529 scenario.name
3530 ))
3531 })?;
3532
3533 let run_uuid = Uuid::parse_str(&req.run_id)
3534 .map_err(|err| invalid_argument(format!("invalid run_id: {err}")))?;
3535 let run = self
3536 .storage
3537 .fetch_run(run_uuid)
3538 .await
3539 .map_err(|err| internal_error(format!("failed to fetch run: {err}")))?
3540 .ok_or_else(|| Status::not_found("run not found"))?;
3541 let steps = self
3542 .storage
3543 .fetch_steps_for_run(run_uuid)
3544 .await
3545 .map_err(|err| internal_error(format!("failed to fetch steps: {err}")))?;
3546 let last_event = self
3547 .storage
3548 .fetch_last_outbox_event(run_uuid)
3549 .await
3550 .map_err(|err| internal_error(format!("failed to fetch run events: {err}")))?;
3551
3552 let previous = self
3553 .storage
3554 .latest_eval_result(scenario.scenario_id)
3555 .await
3556 .map_err(|err| {
3557 internal_error(format!("failed to fetch previous eval result: {err}"))
3558 })?;
3559 let previous_metrics = previous
3560 .as_ref()
3561 .filter(|result| result.run_id != run_uuid)
3562 .map(|result| &result.metrics);
3563
3564 let commit = req.commit.trim();
3565 let commit = if commit.is_empty() {
3566 None
3567 } else {
3568 Some(commit.to_string())
3569 };
3570
3571 let (outcome, metrics) = compute_eval_metrics(
3572 &spec,
3573 &run,
3574 &steps,
3575 last_event.as_ref().map(|event| event.created_at),
3576 previous_metrics,
3577 commit.clone(),
3578 )
3579 .map_err(|err| internal_error(format!("failed to compute eval metrics: {err}")))?;
3580
3581 let stored = self
3582 .storage
3583 .insert_eval_result(scenario.scenario_id, run_uuid, &outcome, metrics)
3584 .await
3585 .map_err(|err| internal_error(format!("failed to persist eval result: {err}")))?;
3586
3587 let proto = build_eval_result_proto(stored)?;
3588 Ok(Response::new(runtime::RecordEvalResultResponse {
3589 result: Some(proto),
3590 }))
3591 }
3592
3593 async fn check_change_gate(
3594 &self,
3595 request: Request<runtime::CheckChangeGateRequest>,
3596 ) -> Result<Response<runtime::CheckChangeGateResponse>, Status> {
3597 let metadata = request.metadata().clone();
3598 let role = self.require_role(&metadata, Role::Writer).await?;
3599 let req = request.into_inner();
3600
3601 let change_id = req.change_id.trim();
3602 if change_id.is_empty() {
3603 return Err(invalid_argument("change_id is required"));
3604 }
3605
3606 let revision = req.revision.trim();
3607 let revision = if revision.is_empty() {
3608 None
3609 } else {
3610 Some(revision.to_string())
3611 };
3612
3613 let actor_header = Self::actor_from_metadata(&metadata, role);
3614 let actor = if req.actor.trim().is_empty() {
3615 actor_header
3616 } else {
3617 req.actor.trim().to_string()
3618 };
3619
3620 let diffs = req
3621 .diffs
3622 .into_iter()
3623 .map(|diff| GateDiff {
3624 path: diff.path,
3625 lines_added: diff.lines_added,
3626 lines_deleted: diff.lines_deleted,
3627 novelty_score: diff.novelty_score,
3628 risk_tags: diff.risk_tags,
3629 component: if diff.component.is_empty() {
3630 None
3631 } else {
3632 Some(diff.component)
3633 },
3634 })
3635 .collect();
3636
3637 let coverage = req
3638 .coverage
3639 .map(|report| GateCoverageReport {
3640 overall_ratio: report.overall_ratio,
3641 minimum_ratio: report.minimum_ratio,
3642 components: report
3643 .components
3644 .into_iter()
3645 .map(|component| GateCoverageComponent {
3646 name: component.name,
3647 coverage_ratio: component.coverage_ratio,
3648 required_ratio: component.required_ratio,
3649 })
3650 .collect(),
3651 })
3652 .unwrap_or_default();
3653
3654 let evals = req
3655 .evals
3656 .into_iter()
3657 .map(|metric| GateEvalMetric {
3658 name: metric.name,
3659 slug: if metric.slug.is_empty() {
3660 None
3661 } else {
3662 Some(metric.slug)
3663 },
3664 score: metric.score,
3665 threshold: metric.threshold,
3666 direction: match runtime::MetricDirection::try_from(metric.direction) {
3667 Ok(runtime::MetricDirection::LowerIsBetter) => {
3668 GateMetricDirection::LowerIsBetter
3669 }
3670 _ => GateMetricDirection::HigherIsBetter,
3671 },
3672 weight: metric.weight,
3673 })
3674 .collect();
3675
3676 let budgets = req
3677 .budgets
3678 .into_iter()
3679 .map(|budget| GateBudgetSignal {
3680 name: budget.name,
3681 actual: budget.actual,
3682 limit: budget.limit,
3683 })
3684 .collect();
3685
3686 let telemetry_value = struct_to_value(req.telemetry);
3687 let metadata_value = struct_to_value(req.metadata);
3688
3689 let gate_request = GateRequest {
3690 change_id: change_id.to_string(),
3691 gate_id: None,
3692 revision: revision.clone(),
3693 diffs,
3694 coverage,
3695 evals,
3696 budgets,
3697 telemetry: telemetry_value.clone(),
3698 metadata: metadata_value.clone(),
3699 actor: Some(actor.clone()),
3700 };
3701
3702 let mut decision = self
3703 .changeops
3704 .evaluate(&gate_request)
3705 .map_err(|err| invalid_argument(err.to_string()))?;
3706
3707 let gate_id = Uuid::new_v4();
3708 let mut stored_request = gate_request.clone();
3709 stored_request.gate_id = Some(gate_id);
3710
3711 let decision_payload = decision.to_value(&stored_request);
3712 let decision_bytes = serde_json::to_vec(&decision_payload).map_err(|err| {
3713 internal_error(format!("failed to serialise change gate decision: {err}"))
3714 })?;
3715
3716 let artifact_meta = self
3717 .storage
3718 .artifacts()
3719 .put(
3720 Bytes::from(decision_bytes),
3721 "application/json",
3722 json!({
3723 "kind": "change_gate_decision",
3724 "change_id": change_id,
3725 "effect": decision.effect.as_str(),
3726 "gate_id": gate_id.to_string(),
3727 }),
3728 )
3729 .await
3730 .map_err(|err| {
3731 internal_error(format!("failed to persist change gate artifact: {err}"))
3732 })?;
3733
3734 let mut attestation_set: HashSet<Uuid> =
3735 attestation_ids_from_value(&stored_request.telemetry)
3736 .into_iter()
3737 .collect();
3738 attestation_set.extend(attestation_ids_from_value(&decision_payload));
3739 let attestation_refs: Vec<Uuid> = attestation_set.into_iter().collect();
3740
3741 if !attestation_refs.is_empty() && self.transparency_scope.includes_gates() {
3742 let artifact_sha = hex::encode(artifact_meta.sha256);
3743 if self.transparency_writer_enabled {
3744 let payload =
3745 models::TransparencyJobPayload::ReleaseBundle(models::ReleaseBundlePayload {
3746 change_id: change_id.to_string(),
3747 gate_id,
3748 artifact_sha256: artifact_sha.clone(),
3749 attestation_ids: attestation_refs.clone(),
3750 metadata: decision.metadata.clone(),
3751 });
3752 match self.storage.enqueue_transparency_job(payload).await {
3753 Ok(job_id) => {
3754 let queued = json!({
3755 "status": "queued",
3756 "job_id": job_id,
3757 "artifact_sha256": artifact_sha,
3758 "attestation_ids": attestation_refs.iter().map(|id| id.to_string()).collect::<Vec<_>>(),
3759 });
3760 if let Value::Object(mut meta_obj) = decision.metadata.take() {
3761 meta_obj.insert("scitt_entry".to_string(), queued);
3762 decision.metadata = Value::Object(meta_obj);
3763 } else {
3764 decision.metadata = json!({ "scitt_entry": queued });
3765 }
3766 }
3767 Err(err) => {
3768 warn!(
3769 change = change_id,
3770 gate = %gate_id,
3771 error = %err,
3772 "failed to enqueue transparency job; falling back to synchronous publish"
3773 );
3774 self.publish_scitt_entry(
3775 change_id,
3776 gate_id,
3777 &artifact_sha,
3778 &attestation_refs,
3779 &mut decision.metadata,
3780 )
3781 .await?;
3782 }
3783 }
3784 } else {
3785 self.publish_scitt_entry(
3786 change_id,
3787 gate_id,
3788 &artifact_sha,
3789 &attestation_refs,
3790 &mut decision.metadata,
3791 )
3792 .await?;
3793 }
3794 }
3795
3796 let stored_gate = self
3797 .storage
3798 .insert_change_gate(models::NewChangeGate {
3799 gate_id,
3800 change_id: change_id.to_string(),
3801 revision: revision.clone(),
3802 effect: decision.effect.as_str().to_string(),
3803 reasons: serde_json::to_value(&decision.reasons).map_err(|err| {
3804 internal_error(format!("failed to serialise change gate reasons: {err}"))
3805 })?,
3806 followups: serde_json::to_value(&decision.followups).map_err(|err| {
3807 internal_error(format!("failed to serialise change gate followups: {err}"))
3808 })?,
3809 scorecard: serde_json::to_value(&decision.scorecard).map_err(|err| {
3810 internal_error(format!("failed to serialise change gate scorecard: {err}"))
3811 })?,
3812 decided_at: decision.decided_at,
3813 decided_by: Some(actor.clone()),
3814 metadata: decision.metadata.clone(),
3815 input: serde_json::to_value(&stored_request).map_err(|err| {
3816 internal_error(format!("failed to serialise change gate request: {err}"))
3817 })?,
3818 telemetry: stored_request.telemetry.clone(),
3819 artifact_sha256: Some(artifact_meta.sha256),
3820 })
3821 .await
3822 .map_err(|err| {
3823 internal_error(format!("failed to persist change gate decision: {err}"))
3824 })?;
3825
3826 let audit_payload = decision_payload.clone();
3827 self.audit
3828 .append(&actor, None, "change_gate.check", &audit_payload)
3829 .await
3830 .map_err(|err| {
3831 internal_error(format!(
3832 "failed to append audit trail for change gate: {err}"
3833 ))
3834 })?;
3835
3836 tracing::info!(
3837 target: "fleetforge.changeops",
3838 gate = %gate_id,
3839 change = %change_id,
3840 effect = decision.effect.as_str(),
3841 reasons = ?decision.reasons,
3842 "change gate evaluated"
3843 );
3844
3845 let proto_decision = build_proto_change_gate(
3846 stored_gate.gate_id,
3847 change_id,
3848 revision.as_deref(),
3849 &decision,
3850 &stored_gate.telemetry,
3851 &[],
3852 )?;
3853
3854 Ok(Response::new(runtime::CheckChangeGateResponse {
3855 decision: Some(proto_decision),
3856 }))
3857 }
3858
3859 async fn record_gate_followup(
3860 &self,
3861 request: Request<runtime::RecordGateFollowupRequest>,
3862 ) -> Result<Response<runtime::RecordGateFollowupResponse>, Status> {
3863 let metadata = request.metadata().clone();
3864 let role = self.require_role(&metadata, Role::Writer).await?;
3865 let req = request.into_inner();
3866
3867 let gate_id = Uuid::parse_str(&req.gate_id)
3868 .map_err(|err| invalid_argument(format!("invalid gate_id: {err}")))?;
3869
3870 let gate = self
3871 .storage
3872 .fetch_change_gate(gate_id)
3873 .await
3874 .map_err(|err| internal_error(format!("failed to fetch change gate: {err}")))?
3875 .ok_or_else(|| Status::not_found("change gate not found"))?;
3876
3877 let actor_header = Self::actor_from_metadata(&metadata, role);
3878 let actor = if req.actor.trim().is_empty() {
3879 actor_header
3880 } else {
3881 req.actor.trim().to_string()
3882 };
3883
3884 let outcome = runtime::FollowupOutcome::try_from(req.outcome)
3885 .unwrap_or(runtime::FollowupOutcome::Approved);
3886 let outcome_gate = proto_followup_outcome(outcome);
3887
3888 let followup_id = Uuid::new_v4();
3889 let note = req.note;
3890 let details_value = struct_to_value(req.details);
3891
3892 let stored_followup = self
3893 .storage
3894 .insert_change_gate_followup(models::NewChangeGateFollowup {
3895 followup_id,
3896 gate_id,
3897 actor: actor.clone(),
3898 outcome: outcome_gate.as_str().to_string(),
3899 note: note.clone(),
3900 details: details_value.clone(),
3901 })
3902 .await
3903 .map_err(|err| {
3904 internal_error(format!("failed to persist change gate follow-up: {err}"))
3905 })?;
3906
3907 let record_payload = ChangeFollowupRecord {
3908 gate_id,
3909 actor: actor.clone(),
3910 outcome: outcome_gate,
3911 note: note.clone(),
3912 details: stored_followup.details.clone(),
3913 recorded_at: stored_followup.recorded_at,
3914 };
3915
3916 self.audit
3917 .append(
3918 &actor,
3919 None,
3920 "change_gate.followup",
3921 &record_payload.to_value(),
3922 )
3923 .await
3924 .map_err(|err| {
3925 internal_error(format!(
3926 "failed to append change gate follow-up audit: {err}"
3927 ))
3928 })?;
3929
3930 tracing::info!(
3931 target: "fleetforge.changeops",
3932 gate = %gate_id,
3933 followup = %followup_id,
3934 outcome = %stored_followup.outcome,
3935 "change gate follow-up recorded"
3936 );
3937
3938 let proto_followup = gate_followup_record_to_proto(&stored_followup);
3939 Ok(Response::new(runtime::RecordGateFollowupResponse {
3940 followup: Some(proto_followup),
3941 }))
3942 }
3943
3944 async fn list_change_gates(
3945 &self,
3946 request: Request<runtime::ListChangeGatesRequest>,
3947 ) -> Result<Response<runtime::ListChangeGatesResponse>, Status> {
3948 let metadata = request.metadata().clone();
3949 self.require_role(&metadata, Role::Reader).await?;
3950 let req = request.into_inner();
3951
3952 let change_id = req.change_id.trim();
3953 if change_id.is_empty() {
3954 return Err(invalid_argument("change_id is required"));
3955 }
3956
3957 let limit = if req.limit == 0 { 5 } else { req.limit.min(50) } as i64;
3958
3959 let gates = self
3960 .storage
3961 .list_change_gates_for_change(change_id, limit)
3962 .await
3963 .map_err(|err| {
3964 internal_error(format!("failed to fetch change gate decisions: {err}"))
3965 })?;
3966
3967 let mut decisions = Vec::with_capacity(gates.len());
3968 for gate in gates {
3969 let followups = self
3970 .storage
3971 .fetch_change_gate_followups(gate.gate_id)
3972 .await
3973 .map_err(|err| {
3974 internal_error(format!("failed to fetch change gate follow-ups: {err}"))
3975 })?;
3976
3977 let effect = gate_effect_from_str(&gate.effect);
3978 let reasons = parse_reasons(&gate.reasons);
3979 let followup_actions = parse_followup_actions(&gate.followups);
3980 let scorecard = parse_scorecard(&gate.scorecard);
3981
3982 let gate_decision = GateDecision {
3983 effect,
3984 reasons,
3985 followups: followup_actions,
3986 scorecard,
3987 decided_at: gate.decided_at,
3988 metadata: gate.metadata.clone(),
3989 };
3990
3991 let proto_decision = build_proto_change_gate(
3992 gate.gate_id,
3993 &gate.change_id,
3994 gate.revision.as_deref(),
3995 &gate_decision,
3996 &gate.telemetry,
3997 &followups,
3998 )?;
3999 decisions.push(proto_decision);
4000 }
4001
4002 Ok(Response::new(runtime::ListChangeGatesResponse {
4003 decisions,
4004 }))
4005 }
4006}
4007
4008#[instrument(skip_all)]
4010pub async fn start_api_server(
4011 storage: Arc<Storage>,
4012 audit: Arc<Audit>,
4013 metadata: Arc<RuntimeMetadata>,
4014) -> Result<()> {
4015 let auth = Arc::new(AuthHandle::from_env().await?);
4016
4017 let endpoints = resolve_endpoint_config()?;
4018 let cors_config = cors_config_from_env();
4019 let transparency = transparency_log_from_env(Arc::clone(&storage))?;
4020 let transparency_scope = transparency_scope_from_env();
4021 let transparency_writer_enabled = transparency_writer_enabled_flag();
4022 let writer_interval = transparency_writer_interval();
4023
4024 if transparency_writer_enabled && !matches!(transparency_scope, TransparencyScope::Disabled) {
4025 info!(
4026 "transparency writer enabled (interval={}s)",
4027 writer_interval.as_secs()
4028 );
4029 let writer_storage = Arc::clone(&storage);
4030 let writer_log = Arc::clone(&transparency);
4031 tokio::spawn(async move {
4032 let writer = TransparencyWriter::new(writer_storage, writer_log, writer_interval);
4033 writer.run().await;
4034 });
4035 }
4036
4037 let cors_summary = cors_config
4038 .as_ref()
4039 .map(|cfg| {
4040 if cfg.display.is_empty() {
4041 "[]".to_string()
4042 } else {
4043 cfg.display.join(", ")
4044 }
4045 })
4046 .unwrap_or_else(|| "disabled".to_string());
4047
4048 info!(
4049 grpc_endpoint = %format!("http://{}", endpoints.grpc_addr),
4050 http_endpoint = %format!("http://{}", endpoints.http_addr),
4051 cors_allowlist = %cors_summary,
4052 "runtime API configuration ready"
4053 );
4054
4055 if endpoints.combined {
4056 run_combined_server(
4057 endpoints.grpc_addr,
4058 storage,
4059 auth,
4060 audit,
4061 transparency,
4062 transparency_writer_enabled,
4063 transparency_scope,
4064 metadata,
4065 cors_config,
4066 )
4067 .await
4068 } else {
4069 let grpc_storage = Arc::clone(&storage);
4070 let http_storage = Arc::clone(&storage);
4071 let http_auth = Arc::clone(&auth);
4072 let grpc_transparency = Arc::clone(&transparency);
4073 let version_meta = Arc::clone(&metadata);
4074 tokio::try_join!(
4075 run_grpc_server(
4076 endpoints.grpc_addr,
4077 grpc_storage,
4078 Arc::clone(&auth),
4079 Arc::clone(&audit),
4080 grpc_transparency,
4081 transparency_writer_enabled,
4082 transparency_scope,
4083 cors_config.clone(),
4084 ),
4085 run_http_server(
4086 endpoints.http_addr,
4087 http_storage,
4088 http_auth,
4089 Arc::clone(&audit),
4090 version_meta,
4091 cors_config,
4092 ),
4093 )?;
4094 Ok(())
4095 }
4096}
4097
4098#[derive(Clone, Copy, Debug)]
4099struct EndpointConfig {
4100 grpc_addr: SocketAddr,
4101 http_addr: SocketAddr,
4102 combined: bool,
4103}
4104
4105fn resolve_endpoint_config() -> Result<EndpointConfig> {
4106 if let Ok(port_raw) = std::env::var("PORT") {
4107 let port: u16 = port_raw.parse().context("invalid PORT value")?;
4108 let addr = SocketAddr::from(([0, 0, 0, 0], port));
4109 return Ok(EndpointConfig {
4110 grpc_addr: addr,
4111 http_addr: addr,
4112 combined: true,
4113 });
4114 }
4115
4116 let grpc_addr: SocketAddr = std::env::var("FLEETFORGE_API_ADDR")
4117 .unwrap_or_else(|_| "0.0.0.0:50051".to_string())
4118 .parse()
4119 .context("invalid FLEETFORGE_API_ADDR")?;
4120 let http_addr: SocketAddr = std::env::var("FLEETFORGE_HTTP_ADDR")
4121 .unwrap_or_else(|_| "0.0.0.0:8080".to_string())
4122 .parse()
4123 .context("invalid FLEETFORGE_HTTP_ADDR")?;
4124
4125 Ok(EndpointConfig {
4126 grpc_addr,
4127 http_addr,
4128 combined: grpc_addr == http_addr,
4129 })
4130}
4131
4132#[instrument(skip_all)]
4133async fn run_combined_server(
4134 addr: SocketAddr,
4135 storage: Arc<Storage>,
4136 auth: Arc<AuthHandle>,
4137 audit: Arc<Audit>,
4138 transparency: Arc<dyn TransparencyLog>,
4139 transparency_writer_enabled: bool,
4140 transparency_scope: TransparencyScope,
4141 metadata: Arc<RuntimeMetadata>,
4142 cors: Option<CorsConfig>,
4143) -> Result<()> {
4144 let listener = TcpListener::bind(addr)
4145 .await
4146 .with_context(|| format!("failed to bind listener on {addr}"))?;
4147
4148 let attestation_vault: Arc<dyn AttestationVault> = Arc::new(ObjectStoreAttestationVault::new(
4149 storage.object_store(),
4150 storage.pool().clone(),
4151 ));
4152 let runtime_api = RuntimeApiService::new(
4153 Arc::clone(&storage),
4154 Arc::clone(&auth),
4155 Arc::clone(&audit),
4156 Arc::clone(&attestation_vault),
4157 Arc::clone(&transparency),
4158 transparency_writer_enabled,
4159 transparency_scope,
4160 );
4161 let tap_api = TapApiService::new(Arc::clone(&storage), Arc::clone(&auth));
4162
4163 let (mut health_reporter, health_service) = health_reporter();
4164 health_reporter
4165 .set_service_status(
4166 "fleetforge.runtime.v1.RuntimeService",
4167 tonic_health::ServingStatus::Serving,
4168 )
4169 .await; health_reporter
4171 .set_service_status(
4172 "fleetforge.tap.v2.TapService",
4173 tonic_health::ServingStatus::Serving,
4174 )
4175 .await;
4176 health_reporter
4177 .set_service_status("", tonic_health::ServingStatus::Serving)
4178 .await;
4179
4180 let runtime_service = RuntimeServiceServer::new(runtime_api);
4181 let runtime_service = tonic_web::enable(runtime_service);
4182 let tap_service = TapServiceServer::new(tap_api);
4183 let tap_service = tonic_web::enable(tap_service);
4184
4185 let reflection = ReflectionBuilder::configure()
4186 .register_encoded_file_descriptor_set(runtime::FILE_DESCRIPTOR_SET)
4187 .build()
4188 .context("failed to build reflection service")?;
4189
4190 let cors_layer = cors.as_ref().map(|cfg| cfg.to_layer());
4191 let http_router = {
4192 let router = http::router(
4193 Arc::clone(&storage),
4194 Arc::clone(&auth),
4195 Arc::clone(&audit),
4196 Arc::clone(&metadata),
4197 );
4198 match cors_layer.clone() {
4199 Some(layer) => router.layer(layer.clone()),
4200 None => router,
4201 }
4202 };
4203 let mut builder = Server::builder().accept_http1(true);
4204
4205 let grpc_service = builder
4206 .add_service(health_service)
4207 .add_service(reflection)
4208 .add_service(runtime_service)
4209 .add_service(tap_service)
4210 .into_service();
4211
4212 let grpc_service = BoxCloneService::new(grpc_service);
4213
4214 info!(%addr, "starting runtime API server (single listener)");
4215
4216 let make_service = make_service_fn(move |_conn| {
4217 let http_router = http_router.clone();
4218 let grpc_service = grpc_service.clone();
4219
4220 async move {
4221 Ok::<_, Infallible>(service_fn(move |req: HyperRequest<Body>| {
4222 let http_router = http_router.clone();
4223 let mut grpc_service = grpc_service.clone();
4224
4225 async move {
4226 if is_grpc_request(&req) {
4227 tower::Service::call(&mut grpc_service, req).await
4228 } else {
4229 let router = http_router.clone();
4230 let response = match router.oneshot(req).await {
4231 Ok(resp) => resp,
4232 Err(err) => match err {},
4233 }
4234 .map(|body| {
4235 body.map_err(|err| Status::internal(err.to_string()))
4236 .boxed_unsync()
4237 });
4238 Ok::<_, Box<dyn std::error::Error + Send + Sync>>(response)
4239 }
4240 }
4241 }))
4242 }
4243 });
4244
4245 let std_listener = listener
4246 .into_std()
4247 .context("failed to convert listener for hyper server")?;
4248 HyperServer::from_tcp(std_listener)
4249 .context("failed to build runtime API server")?
4250 .serve(make_service)
4251 .await
4252 .context("runtime API server exited unexpectedly")
4253}
4254
4255fn is_grpc_request(req: &HyperRequest<Body>) -> bool {
4256 if let Some(content_type) = req
4257 .headers()
4258 .get(header::CONTENT_TYPE)
4259 .and_then(|value| value.to_str().ok())
4260 {
4261 let lower = content_type.to_ascii_lowercase();
4262 if lower.starts_with("application/grpc")
4263 || lower.starts_with("application/grpc-web")
4264 || lower.starts_with("application/connect+")
4265 {
4266 return true;
4267 }
4268 }
4269
4270 let path = req.uri().path();
4271 path.starts_with("/fleetforge.runtime.v1.")
4272 || path.starts_with("/fleetforge.tap.v2.")
4273 || path.starts_with("/grpc.health.v1.")
4274 || path.starts_with("/grpc.reflection.v1alpha.")
4275}
4276
4277#[instrument(skip_all)]
4278async fn run_grpc_server(
4279 addr: SocketAddr,
4280 storage: Arc<Storage>,
4281 auth: Arc<AuthHandle>,
4282 audit: Arc<Audit>,
4283 transparency: Arc<dyn TransparencyLog>,
4284 transparency_writer_enabled: bool,
4285 transparency_scope: TransparencyScope,
4286 cors: Option<CorsConfig>,
4287) -> Result<()> {
4288 let attestation_vault: Arc<dyn AttestationVault> = Arc::new(ObjectStoreAttestationVault::new(
4289 storage.object_store(),
4290 storage.pool().clone(),
4291 ));
4292 let runtime_api = RuntimeApiService::new(
4293 Arc::clone(&storage),
4294 Arc::clone(&auth),
4295 Arc::clone(&audit),
4296 Arc::clone(&attestation_vault),
4297 transparency,
4298 transparency_writer_enabled,
4299 transparency_scope,
4300 );
4301 let tap_api = TapApiService::new(storage, auth);
4302
4303 let (mut health_reporter, health_service) = health_reporter();
4304 let reflection = ReflectionBuilder::configure()
4305 .register_encoded_file_descriptor_set(runtime::FILE_DESCRIPTOR_SET)
4306 .build()
4307 .context("failed to build reflection service")?;
4308
4309 health_reporter
4310 .set_service_status(
4311 "fleetforge.runtime.v1.RuntimeService",
4312 tonic_health::ServingStatus::Serving,
4313 )
4314 .await; health_reporter
4316 .set_service_status(
4317 "fleetforge.tap.v2.TapService",
4318 tonic_health::ServingStatus::Serving,
4319 )
4320 .await;
4321 health_reporter
4322 .set_service_status("", tonic_health::ServingStatus::Serving)
4323 .await;
4324
4325 info!(%addr, "starting runtime gRPC API server");
4326
4327 let runtime_service = RuntimeServiceServer::new(runtime_api);
4328 let runtime_service = tonic_web::enable(runtime_service);
4329 let tap_service = TapServiceServer::new(tap_api);
4330 let tap_service = tonic_web::enable(tap_service);
4331 let cors_layer = cors.as_ref().map(|cfg| cfg.to_layer());
4332 Server::builder()
4333 .accept_http1(true)
4334 .add_service(health_service)
4335 .add_service(reflection)
4336 .add_service(runtime_service)
4337 .add_service(tap_service)
4338 .serve(addr)
4339 .await
4340 .context("runtime gRPC API server exited unexpectedly")
4341}
4342
4343#[instrument(skip_all)]
4344async fn run_http_server(
4345 addr: SocketAddr,
4346 storage: Arc<Storage>,
4347 auth: Arc<AuthHandle>,
4348 audit: Arc<Audit>,
4349 metadata: Arc<RuntimeMetadata>,
4350 cors: Option<CorsConfig>,
4351) -> Result<()> {
4352 let listener = TcpListener::bind(addr)
4353 .await
4354 .with_context(|| format!("failed to bind HTTP listener on {addr}"))?;
4355
4356 let cors_layer = cors.as_ref().map(|cfg| cfg.to_layer());
4358 let router = http::router(storage, auth, audit, metadata);
4359 let app = match cors_layer {
4360 Some(layer) => router.layer(layer),
4361 None => router,
4362 };
4363
4364 info!(%addr, "starting runtime HTTP API server");
4365
4366 let std_listener = listener
4367 .into_std()
4368 .context("failed to convert HTTP listener for hyper server")?;
4369 HyperServer::from_tcp(std_listener)
4370 .context("failed to build runtime HTTP server")?
4371 .serve(app.into_make_service())
4372 .await
4373 .context("runtime HTTP API server exited unexpectedly")
4374}
4375
4376#[derive(Clone, Debug)]
4377struct CorsConfig {
4378 origins: Vec<HeaderValue>,
4379 display: Vec<String>,
4380}
4381
4382impl CorsConfig {
4383 fn to_layer(&self) -> CorsLayer {
4384 CorsLayer::new()
4385 .allow_origin(self.origins.clone())
4386 .allow_methods([Method::GET, Method::POST, Method::OPTIONS])
4387 .allow_headers([header::CONTENT_TYPE, header::AUTHORIZATION])
4388 }
4389}
4390
4391fn cors_config_from_env() -> Option<CorsConfig> {
4392 let raw = std::env::var("FLEETFORGE_CORS_ALLOW_ORIGINS").ok()?;
4393 let mut origins = Vec::new();
4394 let mut display = Vec::new();
4395 for origin in raw.split(',') {
4396 let trimmed = origin.trim();
4397 if trimmed.is_empty() {
4398 continue;
4399 }
4400 match HeaderValue::from_str(trimmed) {
4401 Ok(value) => {
4402 origins.push(value);
4403 display.push(trimmed.to_string());
4404 }
4405 Err(err) => {
4406 warn!(origin = %trimmed, %err, "invalid origin in FLEETFORGE_CORS_ALLOW_ORIGINS")
4407 }
4408 }
4409 }
4410
4411 if origins.is_empty() {
4412 warn!(
4413 "no valid origins parsed from FLEETFORGE_CORS_ALLOW_ORIGINS; CORS will remain disabled"
4414 );
4415 return None;
4416 }
4417
4418 Some(CorsConfig { origins, display })
4419}
4420
4421fn transparency_writer_enabled_flag() -> bool {
4422 let enabled = matches!(
4423 std::env::var(TRANSPARENCY_WRITER_FLAG)
4424 .ok()
4425 .map(|value| value.trim().to_ascii_lowercase())
4426 .unwrap_or_default()
4427 .as_str(),
4428 "1" | "true" | "yes" | "on"
4429 );
4430 if enabled && !feature_allowed(LicensedFeature::ScittTransparency) {
4431 warn!(
4432 "FLEETFORGE_TRANSPARENCY_WRITER=1 but the current license tier does not include SCITT publishing; disabling writer"
4433 );
4434 return false;
4435 }
4436 enabled
4437}
4438
4439fn transparency_scope_from_env() -> TransparencyScope {
4440 match std::env::var("FLEETFORGE_TRANSPARENCY_SCOPE")
4441 .ok()
4442 .map(|value| value.trim().to_ascii_lowercase())
4443 .as_deref()
4444 {
4445 Some("artifacts") => TransparencyScope::Artifacts,
4446 Some("runs") => TransparencyScope::Runs,
4447 Some("gates") => TransparencyScope::Gates,
4448 Some("disabled") => TransparencyScope::Disabled,
4449 _ => TransparencyScope::Gates,
4450 }
4451}
4452
4453fn transparency_writer_interval() -> Duration {
4454 let interval = std::env::var(TRANSPARENCY_WRITER_INTERVAL_ENV)
4455 .ok()
4456 .and_then(|value| value.parse::<u64>().ok())
4457 .unwrap_or(TRANSPARENCY_WRITER_DEFAULT_INTERVAL_SECS);
4458 Duration::from_secs(interval.max(TRANSPARENCY_WRITER_INTERVAL_MIN_SECS))
4459}
4460
4461fn parse_dag_json(input: &str) -> Result<Value, Status> {
4462 serde_json::from_str(input)
4463 .map_err(|err| invalid_argument(format!("invalid dag_json payload: {err}")))
4464}
4465
4466fn extract_priority(step_obj: &Map<String, Value>) -> i16 {
4467 step_obj
4468 .get("policy")
4469 .and_then(Value::as_object)
4470 .and_then(|policy| policy.get("priority"))
4471 .and_then(Value::as_i64)
4472 .unwrap_or(0)
4473 .clamp(i16::MIN as i64, i16::MAX as i64) as i16
4474}
4475
4476fn build_run_detail(
4477 run: models::Run,
4478 steps: Vec<models::Step>,
4479) -> Result<runtime::RunDetail, Status> {
4480 let (inputs_value, labels_map) = unpack_input_ctx(&run.input_ctx);
4481 let breakpoint_specs = extract_breakpoint_specs(&run.input_ctx);
4482 let inputs_struct = match &inputs_value {
4483 Value::Null => None,
4484 Value::Object(map) if map.is_empty() => None,
4485 Value::Object(_) => Some(json_to_prost_struct(&inputs_value).map_err(|err| {
4486 internal_error(format!("failed to encode inputs for response: {err}"))
4487 })?),
4488 _ => None,
4489 };
4490
4491 let dag_json = serde_json::to_string(&run.dag_json).map_err(|err| {
4492 internal_error(format!("failed to serialise dag_json for response: {err}"))
4493 })?;
4494
4495 let run_spec = runtime::RunSpec {
4496 dag_json,
4497 inputs: inputs_struct,
4498 seed: run.seed,
4499 labels: labels_map,
4500 breakpoints: breakpoint_specs,
4501 };
4502
4503 let step_summaries = steps
4504 .into_iter()
4505 .map(step_to_summary)
4506 .collect::<Result<Vec<_>, Status>>()?;
4507
4508 Ok(runtime::RunDetail {
4509 run_id: run.run_id.to_string(),
4510 status: map_run_status(run.status) as i32,
4511 spec: Some(run_spec),
4512 created_at: Some(to_timestamp(run.created_at)),
4513 updated_at: None,
4514 steps: step_summaries,
4515 })
4516}
4517
4518fn unpack_input_ctx(ctx: &Value) -> (Value, HashMap<String, String>) {
4519 match ctx {
4520 Value::Object(map) => {
4521 let inputs = map
4522 .get("inputs")
4523 .cloned()
4524 .unwrap_or_else(|| Value::Object(Map::new()));
4525 let labels = map
4526 .get("labels")
4527 .map(labels_value_to_map)
4528 .unwrap_or_default();
4529 (inputs, labels)
4530 }
4531 other => (other.clone(), HashMap::new()),
4532 }
4533}
4534
4535fn extract_breakpoint_specs(ctx: &Value) -> Vec<String> {
4536 let Value::Object(map) = ctx else {
4537 return Vec::new();
4538 };
4539 let Some(Value::Object(debug)) = map.get("debug") else {
4540 return Vec::new();
4541 };
4542
4543 if let Some(Value::Array(specs)) = debug.get("breakpoint_specs") {
4544 return specs
4545 .iter()
4546 .filter_map(|value| value.as_str().map(|s| s.to_string()))
4547 .collect();
4548 }
4549
4550 if let Some(Value::Array(entries)) = debug.get("breakpoints") {
4551 return entries
4552 .iter()
4553 .filter_map(|entry| {
4554 entry
4555 .get("spec")
4556 .and_then(Value::as_str)
4557 .map(|s| s.to_string())
4558 })
4559 .collect();
4560 }
4561
4562 Vec::new()
4563}
4564
4565fn load_debug_breakpoints(debug: &Map<String, Value>) -> Vec<(String, Uuid)> {
4566 debug
4567 .get("breakpoints")
4568 .and_then(Value::as_array)
4569 .map(|items| {
4570 items
4571 .iter()
4572 .filter_map(|entry| {
4573 let obj = entry.as_object()?;
4574 let spec = obj.get("spec")?.as_str()?.to_string();
4575 let step_id_str = obj.get("step_id")?.as_str()?;
4576 let step_id = Uuid::parse_str(step_id_str).ok()?;
4577 Some((spec, step_id))
4578 })
4579 .collect()
4580 })
4581 .unwrap_or_default()
4582}
4583
4584fn store_debug_breakpoints(debug: &mut Map<String, Value>, entries: &[(String, Uuid)]) {
4585 let list = entries
4586 .iter()
4587 .map(|(spec, step_id)| {
4588 let mut obj = Map::new();
4589 obj.insert("spec".to_string(), Value::String(spec.clone()));
4590 obj.insert("step_id".to_string(), Value::String(step_id.to_string()));
4591 Value::Object(obj)
4592 })
4593 .collect::<Vec<_>>();
4594 debug.insert("breakpoints".to_string(), Value::Array(list));
4595
4596 let specs = entries
4597 .iter()
4598 .map(|(spec, _)| Value::String(spec.clone()))
4599 .collect::<Vec<_>>();
4600 debug.insert("breakpoint_specs".to_string(), Value::Array(specs));
4601}
4602
4603fn load_slug_map(debug: &Map<String, Value>) -> HashMap<String, Uuid> {
4604 debug
4605 .get("slugs")
4606 .and_then(Value::as_object)
4607 .map(|map| {
4608 map.iter()
4609 .filter_map(|(slug, value)| {
4610 value
4611 .as_str()
4612 .and_then(|id| Uuid::parse_str(id).ok())
4613 .map(|uuid| (slug.clone(), uuid))
4614 })
4615 .collect()
4616 })
4617 .unwrap_or_default()
4618}
4619
4620fn store_slug_map(debug: &mut Map<String, Value>, entries: &[(String, Uuid)]) {
4621 let mut slugs = Map::new();
4622 for (slug, step_id) in entries {
4623 slugs.insert(slug.clone(), Value::String(step_id.to_string()));
4624 }
4625 debug.insert("slugs".to_string(), Value::Object(slugs));
4626}
4627
4628fn extract_active_pause_step_id(debug: &Map<String, Value>) -> Option<Uuid> {
4629 debug
4630 .get("active_pause")
4631 .and_then(|value| value.as_object())
4632 .and_then(|obj| obj.get("step_id"))
4633 .and_then(Value::as_str)
4634 .and_then(|s| Uuid::parse_str(s).ok())
4635}
4636
4637fn merge_json_object(target: &mut Map<String, Value>, patch: &Map<String, Value>) {
4638 for (key, value) in patch {
4639 target.insert(key.clone(), value.clone());
4640 }
4641}
4642
4643fn collect_step_ids(run: &models::Run) -> HashSet<Uuid> {
4644 run.dag_json
4645 .get("steps")
4646 .and_then(Value::as_array)
4647 .map(|steps| {
4648 steps
4649 .iter()
4650 .filter_map(|step| {
4651 step.get("id")
4652 .and_then(Value::as_str)
4653 .and_then(|s| Uuid::parse_str(s).ok())
4654 })
4655 .collect()
4656 })
4657 .unwrap_or_default()
4658}
4659
4660fn collect_slug_entries(run: &models::Run) -> Vec<(String, Uuid)> {
4661 run.dag_json
4662 .get("steps")
4663 .and_then(Value::as_array)
4664 .map(|steps| {
4665 steps
4666 .iter()
4667 .filter_map(|step| {
4668 let id = step
4669 .get("id")
4670 .and_then(Value::as_str)
4671 .and_then(|s| Uuid::parse_str(s).ok())?;
4672 let slug = step
4673 .get("slug")
4674 .and_then(Value::as_str)
4675 .map(|s| s.trim().to_string())
4676 .filter(|s| !s.is_empty())?;
4677 Some((slug, id))
4678 })
4679 .collect()
4680 })
4681 .unwrap_or_default()
4682}
4683
4684fn build_branch_dag(dag_json: &Value, start: Uuid) -> Result<Value, SubmitRunError> {
4685 let dag_map = dag_json
4686 .as_object()
4687 .cloned()
4688 .ok_or_else(|| SubmitRunError::invalid("dag_json must be a JSON object"))?;
4689 let steps_array = dag_map
4690 .get("steps")
4691 .and_then(Value::as_array)
4692 .ok_or_else(|| SubmitRunError::invalid("dag_json.steps must be an array"))?;
4693 let edges_array = dag_map
4694 .get("edges")
4695 .and_then(Value::as_array)
4696 .ok_or_else(|| SubmitRunError::invalid("dag_json.edges must be an array"))?;
4697
4698 let mut step_defs: HashMap<Uuid, Value> = HashMap::new();
4699 for step in steps_array {
4700 let step_obj = step
4701 .as_object()
4702 .ok_or_else(|| SubmitRunError::invalid("each step must be an object"))?;
4703 let id = step_obj
4704 .get("id")
4705 .and_then(Value::as_str)
4706 .ok_or_else(|| SubmitRunError::invalid("step.id missing"))?;
4707 let step_id = Uuid::parse_str(id)
4708 .map_err(|err| SubmitRunError::invalid(format!("invalid step.id: {err}")))?;
4709 step_defs.insert(step_id, step.clone());
4710 }
4711
4712 if !step_defs.contains_key(&start) {
4713 return Err(SubmitRunError::invalid("from_step references unknown step"));
4714 }
4715
4716 let mut adjacency: HashMap<Uuid, Vec<Uuid>> = HashMap::new();
4717 for edge in edges_array {
4718 let pair = edge
4719 .as_array()
4720 .ok_or_else(|| SubmitRunError::invalid("edges entries must be arrays"))?;
4721 if pair.len() != 2 {
4722 return Err(SubmitRunError::invalid(
4723 "edges entries must contain two members",
4724 ));
4725 }
4726 let from = pair[0]
4727 .as_str()
4728 .ok_or_else(|| SubmitRunError::invalid("edge source must be a string"))?;
4729 let to = pair[1]
4730 .as_str()
4731 .ok_or_else(|| SubmitRunError::invalid("edge target must be a string"))?;
4732 let from_id = Uuid::parse_str(from)
4733 .map_err(|err| SubmitRunError::invalid(format!("invalid edge source: {err}")))?;
4734 let to_id = Uuid::parse_str(to)
4735 .map_err(|err| SubmitRunError::invalid(format!("invalid edge target: {err}")))?;
4736 adjacency.entry(from_id).or_default().push(to_id);
4737 }
4738
4739 let mut queue = VecDeque::new();
4740 let mut visited: HashSet<Uuid> = HashSet::new();
4741 queue.push_back(start);
4742 while let Some(node) = queue.pop_front() {
4743 if !visited.insert(node) {
4744 continue;
4745 }
4746 if let Some(neighbors) = adjacency.get(&node) {
4747 for next in neighbors {
4748 queue.push_back(*next);
4749 }
4750 }
4751 }
4752
4753 let filtered_steps = steps_array
4754 .iter()
4755 .filter_map(|step| {
4756 let id = step
4757 .get("id")
4758 .and_then(Value::as_str)
4759 .and_then(|s| Uuid::parse_str(s).ok());
4760 match id {
4761 Some(step_id) if visited.contains(&step_id) => Some(step.clone()),
4762 _ => None,
4763 }
4764 })
4765 .collect::<Vec<_>>();
4766
4767 let filtered_edges = edges_array
4768 .iter()
4769 .filter(|edge| {
4770 if let Some(pair) = edge.as_array() {
4771 if pair.len() == 2 {
4772 if let (Some(from), Some(to)) = (
4773 pair[0].as_str().and_then(|s| Uuid::parse_str(s).ok()),
4774 pair[1].as_str().and_then(|s| Uuid::parse_str(s).ok()),
4775 ) {
4776 return visited.contains(&from) && visited.contains(&to);
4777 }
4778 }
4779 }
4780 false
4781 })
4782 .cloned()
4783 .collect::<Vec<_>>();
4784
4785 let mut new_dag = dag_json.as_object().cloned().unwrap_or_default();
4786 new_dag.insert("steps".to_string(), Value::Array(filtered_steps));
4787 new_dag.insert("edges".to_string(), Value::Array(filtered_edges));
4788 Ok(Value::Object(new_dag))
4789}
4790
4791fn collect_prompt_packs_json(dag_json: &Value) -> Vec<String> {
4792 match dag_json {
4793 Value::Object(map) => {
4794 let packs = map.get("prompt_packs").or_else(|| map.get("promptPacks"));
4795 match packs {
4796 Some(Value::Array(items)) => items
4797 .iter()
4798 .filter_map(|item| item.as_str().map(|s| s.to_string()))
4799 .collect(),
4800 Some(Value::String(value)) => vec![value.clone()],
4801 _ => Vec::new(),
4802 }
4803 }
4804 _ => Vec::new(),
4805 }
4806}
4807
4808struct DriftComputation {
4809 token_delta: Option<f64>,
4810 cost_delta: Option<f64>,
4811 within_tolerance: bool,
4812}
4813
4814const DEFAULT_TOKEN_TOLERANCE: f64 = 0.0;
4815const DEFAULT_COST_TOLERANCE: f64 = 0.0;
4816const TOKEN_KEYS: &[&str] = &["tokens", "token_count", "total_tokens"];
4817const COST_KEYS: &[&str] = &["cost", "cost_usd", "total_cost"];
4818const SANITIZE_KEYS: &[&str] = &[
4819 "tokens",
4820 "token_count",
4821 "total_tokens",
4822 "cost",
4823 "cost_usd",
4824 "total_cost",
4825];
4826
4827fn wrap_attempt_output(attempt: Option<&models::StepAttempt>, output: &Value) -> Value {
4828 match attempt {
4829 Some(record) => json!({
4830 "attempt": record.attempt,
4831 "status": record.status.as_str(),
4832 "recorded_at": record.recorded_at,
4833 "output": output,
4834 }),
4835 None => output.clone(),
4836 }
4837}
4838
4839fn compute_drift(
4840 recorded: &Value,
4841 current: &Value,
4842 token_tolerance: f64,
4843 cost_tolerance: f64,
4844) -> DriftComputation {
4845 let token_delta = diff_numeric(recorded, current, TOKEN_KEYS);
4846 let cost_delta = diff_numeric(recorded, current, COST_KEYS);
4847
4848 let sanitized_recorded = strip_keys(recorded, SANITIZE_KEYS);
4849 let sanitized_current = strip_keys(current, SANITIZE_KEYS);
4850
4851 let tokens_within = token_delta
4852 .map(|delta| delta.abs() <= token_tolerance)
4853 .unwrap_or(true);
4854 let cost_within = cost_delta
4855 .map(|delta| delta.abs() <= cost_tolerance)
4856 .unwrap_or(true);
4857 let structure_equal = sanitized_recorded == sanitized_current;
4858
4859 DriftComputation {
4860 token_delta,
4861 cost_delta,
4862 within_tolerance: tokens_within && cost_within && structure_equal,
4863 }
4864}
4865
4866fn diff_numeric(recorded: &Value, current: &Value, keys: &[&str]) -> Option<f64> {
4867 let recorded_value = find_numeric(recorded, keys)?;
4868 let current_value = find_numeric(current, keys)?;
4869 Some(current_value - recorded_value)
4870}
4871
4872fn find_numeric(value: &Value, keys: &[&str]) -> Option<f64> {
4873 match value {
4874 Value::Object(map) => {
4875 for key in keys {
4876 if let Some(v) = map.get(*key) {
4877 if let Some(num) = v.as_f64() {
4878 return Some(num);
4879 }
4880 }
4881 }
4882 for v in map.values() {
4883 if let Some(found) = find_numeric(v, keys) {
4884 return Some(found);
4885 }
4886 }
4887 None
4888 }
4889 Value::Array(items) => {
4890 for item in items {
4891 if let Some(found) = find_numeric(item, keys) {
4892 return Some(found);
4893 }
4894 }
4895 None
4896 }
4897 _ => None,
4898 }
4899}
4900
4901fn strip_keys(value: &Value, keys: &[&str]) -> Value {
4902 match value {
4903 Value::Object(map) => {
4904 let mut new_map = serde_json::Map::new();
4905 for (k, v) in map {
4906 if keys.iter().any(|key| key == k) {
4907 continue;
4908 }
4909 new_map.insert(k.clone(), strip_keys(v, keys));
4910 }
4911 Value::Object(new_map)
4912 }
4913 Value::Array(items) => Value::Array(items.iter().map(|v| strip_keys(v, keys)).collect()),
4914 _ => value.clone(),
4915 }
4916}
4917
4918fn step_to_summary(step: models::Step) -> Result<runtime::StepSummary, Status> {
4919 let step_type = step
4920 .spec_json
4921 .get("type")
4922 .and_then(Value::as_str)
4923 .unwrap_or_default()
4924 .to_string();
4925
4926 let output = match step.output_json {
4927 Some(value) => Some(json_to_prost_value(&value).map_err(|err| {
4928 internal_error(format!(
4929 "invalid output payload for step {}: {err}",
4930 step.step_id
4931 ))
4932 })?),
4933 None => None,
4934 };
4935
4936 let error = match step.error_json {
4937 Some(value) => Some(json_to_prost_value(&value).map_err(|err| {
4938 internal_error(format!(
4939 "invalid error payload for step {}: {err}",
4940 step.step_id
4941 ))
4942 })?),
4943 None => None,
4944 };
4945
4946 Ok(runtime::StepSummary {
4947 step_id: step.step_id.to_string(),
4948 idx: step.idx,
4949 r#type: step_type,
4950 status: step.status.as_str().to_string(),
4951 attempt: step.attempt,
4952 leased_at: None,
4953 completed_at: None,
4954 output,
4955 error,
4956 })
4957}
4958
4959fn run_event_from_outbox(event: models::OutboxEvent) -> Result<runtime::RunEvent, Status> {
4960 let payload = json_to_prost_value(&event.payload).map_err(|err| {
4961 internal_error(format!(
4962 "failed to encode outbox payload for event {}: {err}",
4963 event.id
4964 ))
4965 })?;
4966
4967 Ok(runtime::RunEvent {
4968 run_id: event.run_id.to_string(),
4969 step_id: event.step_id.map(|id| id.to_string()).unwrap_or_default(),
4970 kind: event.kind,
4971 payload: Some(payload),
4972 occurred_at: Some(to_timestamp(event.created_at)),
4973 })
4974}
4975
4976fn map_run_status(status: models::RunStatus) -> runtime::RunStatus {
4977 match status {
4978 models::RunStatus::Pending => runtime::RunStatus::Pending,
4979 models::RunStatus::Running => runtime::RunStatus::Running,
4980 models::RunStatus::Succeeded => runtime::RunStatus::Succeeded,
4981 models::RunStatus::Failed => runtime::RunStatus::Failed,
4982 models::RunStatus::Canceled => runtime::RunStatus::Canceled,
4983 models::RunStatus::PausedAtStep => runtime::RunStatus::PausedAtStep,
4984 }
4985}
4986
4987#[derive(Clone, Debug)]
4988pub(crate) struct EvalScenarioSpecData {
4989 pack: String,
4990 slug: String,
4991 title: Option<String>,
4992 description: Option<String>,
4993 run_spec: Value,
4994 expected: Value,
4995}
4996
4997pub(crate) fn parse_eval_spec(spec: &Value) -> anyhow::Result<EvalScenarioSpecData> {
4998 let pack = spec
4999 .get("pack")
5000 .and_then(Value::as_str)
5001 .ok_or_else(|| anyhow!("scenario spec missing 'pack' field"))?
5002 .to_string();
5003 let slug = spec
5004 .get("slug")
5005 .and_then(Value::as_str)
5006 .ok_or_else(|| anyhow!("scenario spec missing 'slug' field"))?
5007 .to_string();
5008 let title = spec
5009 .get("title")
5010 .and_then(Value::as_str)
5011 .map(|value| value.to_string());
5012 let description = spec
5013 .get("description")
5014 .and_then(Value::as_str)
5015 .map(|value| value.to_string());
5016 let run_spec = spec
5017 .get("run_spec")
5018 .cloned()
5019 .ok_or_else(|| anyhow!("scenario spec missing 'run_spec' payload"))?;
5020 if !run_spec.is_object() {
5021 return Err(anyhow!("scenario spec 'run_spec' must be a JSON object"));
5022 }
5023 let expected = spec
5024 .get("expected")
5025 .cloned()
5026 .ok_or_else(|| anyhow!("scenario spec missing 'expected' payload"))?;
5027 if !expected.is_object() {
5028 return Err(anyhow!("scenario spec 'expected' must be a JSON object"));
5029 }
5030
5031 Ok(EvalScenarioSpecData {
5032 pack,
5033 slug,
5034 title,
5035 description,
5036 run_spec,
5037 expected,
5038 })
5039}
5040
5041fn metric_summary(current: f64, previous: Option<f64>) -> Value {
5042 let delta = previous.map(|p| current - p);
5043 json!({
5044 "value": current,
5045 "previous": previous,
5046 "delta": delta,
5047 })
5048}
5049
5050fn extract_metric(metrics: &Value, key: &str) -> Option<f64> {
5051 metrics
5052 .as_object()
5053 .and_then(|map| map.get(key))
5054 .and_then(|entry| entry.get("value"))
5055 .and_then(Value::as_f64)
5056}
5057
5058fn compute_eval_metrics(
5059 spec: &EvalScenarioSpecData,
5060 run: &models::Run,
5061 steps: &[models::Step],
5062 last_event_time: Option<DateTime<Utc>>,
5063 previous_metrics: Option<&Value>,
5064 commit: Option<String>,
5065) -> anyhow::Result<(String, Value)> {
5066 let tokens_actual: f64 = steps
5067 .iter()
5068 .map(|step| {
5069 step.actual_tokens
5070 .map(|value| value as f64)
5071 .unwrap_or_else(|| step.estimated_tokens as f64)
5072 })
5073 .sum();
5074 let cost_actual: f64 = steps
5075 .iter()
5076 .map(|step| step.actual_cost.unwrap_or(step.estimated_cost))
5077 .sum();
5078
5079 let run_end = last_event_time.unwrap_or(run.created_at);
5080 let latency_ms = run_end
5081 .signed_duration_since(run.created_at)
5082 .num_milliseconds()
5083 .max(0) as f64;
5084
5085 let previous_tokens = previous_metrics.and_then(|metrics| extract_metric(metrics, "tokens"));
5086 let previous_cost = previous_metrics.and_then(|metrics| extract_metric(metrics, "cost_usd"));
5087 let previous_latency =
5088 previous_metrics.and_then(|metrics| extract_metric(metrics, "latency_ms"));
5089
5090 let mut differences: Vec<Value> = Vec::new();
5091 if let Some(expected_status) = spec.expected.get("status").and_then(Value::as_str) {
5092 let actual_status = run.status.as_str();
5093 if !expected_status.eq_ignore_ascii_case(actual_status) {
5094 differences.push(json!({
5095 "type": "status",
5096 "expected": expected_status,
5097 "actual": actual_status,
5098 }));
5099 }
5100 }
5101
5102 let mut actual_steps: HashMap<String, Value> = HashMap::with_capacity(steps.len());
5103 for step in steps {
5104 if let Some(output) = step.output_json.clone() {
5105 actual_steps.insert(step.step_id.to_string(), output);
5106 }
5107 }
5108
5109 if let Some(expected_steps) = spec.expected.get("steps").and_then(Value::as_object) {
5110 for (step_id, expected_output) in expected_steps {
5111 match actual_steps.get(step_id) {
5112 Some(actual_output) if actual_output == expected_output => {}
5113 Some(actual_output) => {
5114 differences.push(json!({
5115 "type": "step",
5116 "step_id": step_id,
5117 "expected": expected_output,
5118 "actual": actual_output,
5119 }));
5120 }
5121 None => {
5122 differences.push(json!({
5123 "type": "step_missing",
5124 "step_id": step_id,
5125 "expected": expected_output,
5126 }));
5127 }
5128 }
5129 }
5130 }
5131
5132 let quality_matched = differences.is_empty();
5133 let outcome = if quality_matched { "passed" } else { "failed" }.to_string();
5134
5135 let mut metrics_map = Map::new();
5136 if let Some(commit) = commit {
5137 if !commit.is_empty() {
5138 metrics_map.insert("commit".to_string(), Value::String(commit));
5139 }
5140 }
5141 metrics_map.insert(
5142 "tokens".to_string(),
5143 metric_summary(tokens_actual, previous_tokens),
5144 );
5145 metrics_map.insert(
5146 "cost_usd".to_string(),
5147 metric_summary(cost_actual, previous_cost),
5148 );
5149 metrics_map.insert(
5150 "latency_ms".to_string(),
5151 metric_summary(latency_ms, previous_latency),
5152 );
5153 metrics_map.insert(
5154 "quality".to_string(),
5155 Value::Object({
5156 let mut quality = Map::new();
5157 quality.insert("matched".to_string(), Value::Bool(quality_matched));
5158 quality.insert("differences".to_string(), Value::Array(differences));
5159 quality
5160 }),
5161 );
5162
5163 Ok((outcome, Value::Object(metrics_map)))
5164}
5165
5166fn build_eval_result_proto(result: models::EvalResult) -> Result<runtime::EvalResult, Status> {
5167 let metrics_struct = json_to_prost_struct(&result.metrics)
5168 .map_err(|err| internal_error(format!("invalid eval metrics payload: {err}")))?;
5169 let commit = result
5170 .metrics
5171 .get("commit")
5172 .and_then(Value::as_str)
5173 .unwrap_or("")
5174 .to_string();
5175
5176 Ok(runtime::EvalResult {
5177 result_id: result.result_id.to_string(),
5178 run_id: result.run_id.to_string(),
5179 scenario_id: result.scenario_id.to_string(),
5180 outcome: result.outcome,
5181 commit,
5182 executed_at: Some(to_timestamp(result.executed_at)),
5183 metrics: Some(metrics_struct),
5184 })
5185}
5186
5187fn build_eval_scenario_proto(
5188 scenario: &models::EvalScenario,
5189 spec: &EvalScenarioSpecData,
5190 history: Vec<runtime::EvalResult>,
5191) -> Result<runtime::EvalScenario, Status> {
5192 Ok(runtime::EvalScenario {
5193 scenario_id: scenario.scenario_id.to_string(),
5194 pack: spec.pack.clone(),
5195 slug: spec.slug.clone(),
5196 title: spec.title.clone().unwrap_or_else(|| spec.slug.clone()),
5197 description: spec.description.clone().unwrap_or_default(),
5198 created_at: Some(to_timestamp(scenario.created_at)),
5199 history,
5200 })
5201}
5202
5203#[cfg(test)]
5204mod unit_tests {
5205 use super::*;
5206 use chrono::Utc;
5207 use fleetforge_common::prost_json::prost_value_to_json;
5208 use fleetforge_storage::models;
5209 use serde_json::{json, Map, Value};
5210 use std::collections::{HashMap, VecDeque};
5211 use std::time::Duration;
5212 use tonic::Code;
5213 use uuid::Uuid;
5214
5215 #[test]
5216 fn parse_page_token_supports_defaults() {
5217 assert_eq!(parse_page_token("").expect("empty token"), 0);
5218 assert_eq!(parse_page_token("0").expect("zero token"), 0);
5219 assert!(parse_page_token("abc").is_err());
5220 }
5221
5222 #[test]
5223 fn query_filters_normalizes_inputs() {
5224 let run_id = Uuid::new_v4();
5225 let mut request = QueryEventsRequest::default();
5226 request.run_ids.push(run_id.to_string());
5227 request.kinds.push("Step_Succeeded".to_string());
5228 request.workspace_ids.push("Workspace-1".to_string());
5229 request
5230 .policy_effects
5231 .push(PolicyDecisionEffect::PolicyDecisionEffectDeny as i32);
5232 request.severities.push(EventSeverity::Error as i32);
5233
5234 let filters = QueryFiltersOwned::from_request(&request).expect("filters");
5235 assert_eq!(filters.run_ids, vec![run_id]);
5236 assert_eq!(filters.kinds, vec!["step_succeeded"]);
5237 assert_eq!(filters.workspace_ids, vec!["workspace-1"]);
5238 assert_eq!(
5239 filters.policy_effects,
5240 vec![PolicyDecisionEffect::PolicyDecisionEffectDeny]
5241 );
5242 assert_eq!(filters.severities, vec![EventSeverity::Error]);
5243 }
5244
5245 #[test]
5246 fn matches_filters_respects_workspace_and_kind() {
5247 let run_id = Uuid::new_v4();
5248 let mut request = QueryEventsRequest::default();
5249 request.run_ids.push(run_id.to_string());
5250 request.workspace_ids.push("tenant-x".to_string());
5251 request.kinds.push("run_succeeded".to_string());
5252 let filters = QueryFiltersOwned::from_request(&request).expect("filters");
5253
5254 let payload = json!({"trace": {"run": {"workspace_id": "Tenant-X"}}});
5255 let kind = "run_succeeded";
5256 let severity = EventSeverity::Info;
5257 assert!(
5258 matches_filters(&filters, &payload, kind, severity, None),
5259 "matching workspace should pass filter"
5260 );
5261
5262 let other_payload = json!({"trace": {"run": {"workspace_id": "tenant-y"}}});
5263 assert!(
5264 !matches_filters(&filters, &other_payload, kind, severity, None),
5265 "non-matching workspace should fail filter"
5266 );
5267 }
5268
5269 #[test]
5270 fn parse_dag_json_round_trip() {
5271 let dag = r#"{"steps": []}"#;
5272 let parsed = parse_dag_json(dag).expect("valid JSON");
5273 assert_eq!(parsed["steps"], Value::Array(vec![]));
5274
5275 let err = parse_dag_json("not valid json").expect_err("should reject invalid JSON");
5276 assert_eq!(err.code(), Code::InvalidArgument);
5277 }
5278
5279 #[test]
5280 fn extract_priority_clamps_to_i16() {
5281 let mut policy = Map::new();
5282 policy.insert("priority".to_string(), Value::from(40_000));
5283 let mut step = Map::new();
5284 step.insert("policy".to_string(), Value::Object(policy));
5285 let priority = extract_priority(&step);
5286 assert_eq!(priority, i16::MAX);
5287 }
5288
5289 #[test]
5290 fn extract_priority_defaults_to_zero() {
5291 let step = Map::new();
5292 let priority = extract_priority(&step);
5293 assert_eq!(priority, 0);
5294 }
5295
5296 #[test]
5297 fn extract_priority_handles_negative_overflow() {
5298 let mut policy = Map::new();
5299 policy.insert("priority".to_string(), Value::from(-50_000));
5300 let mut step = Map::new();
5301 step.insert("policy".to_string(), Value::Object(policy));
5302 let priority = extract_priority(&step);
5303 assert_eq!(priority, i16::MIN);
5304 }
5305
5306 #[test]
5307 fn labels_map_round_trip() {
5308 let mut map = HashMap::new();
5309 map.insert("team".to_string(), "infra".to_string());
5310 map.insert("env".to_string(), "dev".to_string());
5311 let value = labels_map_to_value(&map);
5312 let round_trip = labels_value_to_map(&value);
5313 assert_eq!(map, round_trip);
5314 }
5315
5316 #[test]
5317 fn build_run_detail_converts_payloads() {
5318 let run_id = Uuid::new_v4();
5319 let step_id = Uuid::new_v4();
5320 let now = Utc::now();
5321
5322 let run = models::Run {
5323 run_id,
5324 created_at: now,
5325 status: models::RunStatus::Pending,
5326 dag_json: json!({"steps": []}),
5327 input_ctx: json!({
5328 "inputs": {"foo": "bar"},
5329 "labels": {"team": "infra"}
5330 }),
5331 seed: 7,
5332 idempotency_key: Some("abc".into()),
5333 };
5334
5335 let step = models::Step {
5336 run_id,
5337 step_id,
5338 idx: 0,
5339 priority: 0,
5340 spec_json: json!({"id": step_id, "type": "tool", "inputs": {}, "policy": {}}),
5341 status: models::StepStatus::Succeeded,
5342 attempt: 1,
5343 created_at: now,
5344 pending_dependencies: 0,
5345 total_dependencies: 0,
5346 estimated_tokens: 0,
5347 estimated_cost: 0.0,
5348 actual_tokens: Some(10),
5349 actual_cost: Some(0.01),
5350 leased_by: None,
5351 lease_expires_at: None,
5352 output_json: Some(json!({"answer": 42})),
5353 error_json: Some(json!({"message": "none"})),
5354 input_snapshot: None,
5355 output_snapshot: None,
5356 provider: Some("test".into()),
5357 provider_version: Some("1.0".into()),
5358 };
5359
5360 let detail = build_run_detail(run, vec![step]).expect("build run detail");
5361 let spec = detail.spec.expect("spec present");
5362 assert_eq!(spec.dag_json, r#"{"steps":[]}"#);
5363 assert_eq!(spec.seed, 7);
5364 assert_eq!(spec.labels.get("team"), Some(&"infra".to_string()));
5365
5366 let step_summary = &detail.steps[0];
5367 assert_eq!(step_summary.step_id, step_id.to_string());
5368 assert_eq!(step_summary.status, "succeeded");
5369 let output_value = prost_value_to_json(step_summary.output.as_ref().unwrap());
5370 assert_eq!(output_value, json!({"answer": 42}));
5371 }
5372
5373 #[test]
5374 fn subscription_config_parses_filters() {
5375 let run_id = Uuid::new_v4().to_string();
5376 let step_id = Uuid::new_v4().to_string();
5377 let open = SubscribeRunOpen {
5378 run_ids: vec![run_id.clone()],
5379 step_ids: vec![step_id.clone()],
5380 kinds: vec!["step_succeeded".to_string()],
5381 severities: vec![EventSeverity::Warn as i32],
5382 tags: vec!["team:core".to_string()],
5383 since_offset: 10,
5384 batch_size: 32,
5385 max_inflight: 20,
5386 heartbeat_interval_ms: 5_000,
5387 };
5388
5389 let config = SubscriptionConfig::try_from_open(open).expect("valid config");
5390 assert_eq!(config.run_id.to_string(), run_id);
5391 assert!(config
5392 .step_ids
5393 .as_ref()
5394 .expect("step filter")
5395 .contains(&Uuid::parse_str(&step_id).unwrap()));
5396 assert!(config
5397 .kinds
5398 .as_ref()
5399 .expect("kind filter")
5400 .contains("step_succeeded"));
5401 assert!(config
5402 .severities
5403 .as_ref()
5404 .expect("severity filter")
5405 .contains(&EventSeverity::Warn));
5406 assert!(config
5407 .tags
5408 .as_ref()
5409 .expect("tag filter")
5410 .contains("team:core"));
5411 assert_eq!(config.since_offset, 10);
5412 assert_eq!(config.batch_size, 32);
5413 assert_eq!(config.max_inflight, 20);
5414 assert_eq!(config.heartbeat.as_millis(), 5_000);
5415 }
5416
5417 #[test]
5418 fn handle_ack_evicts_inflight_offsets() {
5419 let mut inflight = VecDeque::from(vec![5, 9, 12]);
5420 let mut last_ack = 0;
5421 handle_ack(&mut inflight, &mut last_ack, 9);
5422 assert_eq!(last_ack, 9);
5423 assert_eq!(inflight, VecDeque::from(vec![12]));
5424
5425 handle_ack(&mut inflight, &mut last_ack, 8);
5426 assert_eq!(last_ack, 9, "ack lower than last should be ignored");
5427 assert_eq!(inflight, VecDeque::from(vec![12]));
5428
5429 handle_ack(&mut inflight, &mut last_ack, 15);
5430 assert_eq!(last_ack, 15);
5431 assert!(inflight.is_empty());
5432 }
5433
5434 #[test]
5435 fn passes_filters_applies_tags_and_severities() {
5436 let run_id = Uuid::new_v4();
5437 let step_id = Uuid::new_v4();
5438 let base_config = SubscriptionConfig {
5439 run_id,
5440 step_ids: Some(vec![step_id].into_iter().collect()),
5441 kinds: Some(vec!["step_succeeded".to_string()].into_iter().collect()),
5442 severities: Some(vec![EventSeverity::Info].into_iter().collect()),
5443 tags: Some(vec!["severity:info".to_string()].into_iter().collect()),
5444 cursor: None,
5445 since_offset: 0,
5446 batch_size: 16,
5447 max_inflight: 8,
5448 heartbeat: Duration::from_millis(1_000),
5449 };
5450
5451 let event = models::OutboxEvent {
5452 id: 1,
5453 run_id,
5454 step_id: Some(step_id),
5455 kind: "STEP_SUCCEEDED".to_string(),
5456 payload: json!({
5457 "status": "succeeded"
5458 }),
5459 created_at: Utc::now(),
5460 published_at: None,
5461 };
5462
5463 let severity = derive_severity(&event.kind, &event.payload);
5464 assert_eq!(severity, EventSeverity::Info);
5465 let tags = derive_tags(&event.kind, event.step_id, &event.payload, severity);
5466 assert!(passes_filters(&base_config, &event, severity, &tags));
5467
5468 let bad_config = SubscriptionConfig {
5469 tags: Some(vec!["severity:error".to_string()].into_iter().collect()),
5470 ..base_config
5471 };
5472 assert!(!passes_filters(&bad_config, &event, severity, &tags));
5473 }
5474}
5475
5476#[cfg(test)]
5477mod tests {
5478 use super::*;
5479 use chrono::Utc;
5480 use fleetforge_storage::{ObjectStoreConfig, StorageConfig};
5481 use fleetforge_telemetry::audit::Audit;
5482 use serde_json::json;
5483 use std::collections::HashMap;
5484 use testcontainers::clients::Cli;
5485 use testcontainers::images::postgres::Postgres;
5486 use tokio::time::{sleep, Duration};
5487 use uuid::Uuid;
5488
5489 fn build_request(dag: serde_json::Value) -> runtime::SubmitRunRequest {
5490 build_request_with_key(dag, "")
5491 }
5492
5493 fn build_request_with_key(
5494 dag: serde_json::Value,
5495 idempotency_key: impl Into<String>,
5496 ) -> runtime::SubmitRunRequest {
5497 runtime::SubmitRunRequest {
5498 spec: Some(runtime::RunSpec {
5499 dag_json: serde_json::to_string(&dag).unwrap(),
5500 inputs: None,
5501 seed: 123,
5502 labels: HashMap::new(),
5503 }),
5504 idempotency_key: idempotency_key.into(),
5505 }
5506 }
5507
5508 #[tokio::test]
5509 async fn submit_run_rejects_invalid_step_schema() {
5510 if std::fs::metadata("/var/run/docker.sock").is_err()
5511 && std::env::var("DOCKER_HOST").is_err()
5512 {
5513 eprintln!("Skipping API tests because Docker is unavailable");
5514 return;
5515 }
5516
5517 let docker = Cli::default();
5518 let container = docker.run(Postgres::default());
5519 let port = container.get_host_port_ipv4(5432);
5520 let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5521 sleep(Duration::from_millis(250)).await;
5522
5523 let storage = Arc::new(
5524 Storage::connect(StorageConfig {
5525 database_url: db_url,
5526 max_connections: 2,
5527 connect_timeout: Duration::from_secs(10),
5528 object_store: ObjectStoreConfig::InMemory,
5529 })
5530 .await
5531 .expect("storage connect"),
5532 );
5533 let auth = Arc::new(AuthHandle::disabled());
5534 let audit = Arc::new(Audit::new(storage.pool().clone()));
5535 let attestation_vault: Arc<dyn AttestationVault> =
5536 Arc::new(InMemoryAttestationVault::new());
5537 let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5538 let service = RuntimeApiService::new(
5539 storage,
5540 auth,
5541 audit,
5542 attestation_vault,
5543 transparency,
5544 false,
5545 TransparencyScope::Disabled,
5546 );
5547
5548 let dag = serde_json::json!({
5549 "steps": [
5550 {
5551 "type": "tool",
5552 "inputs": {}
5553 }
5554 ]
5555 });
5556
5557 let request = build_request(dag);
5558 let response = service.submit_run(Request::new(request)).await;
5559 assert!(response.is_err(), "expected schema validation failure");
5560 let status = response.err().unwrap();
5561 assert_eq!(status.code(), tonic::Code::InvalidArgument);
5562 assert!(status.message().contains("step schema validation failed"));
5563 drop(container);
5564 }
5565
5566 #[tokio::test]
5567 async fn submit_run_rejects_missing_seed() {
5568 if std::fs::metadata("/var/run/docker.sock").is_err()
5569 && std::env::var("DOCKER_HOST").is_err()
5570 {
5571 eprintln!("Skipping API tests because Docker is unavailable");
5572 return;
5573 }
5574
5575 let docker = Cli::default();
5576 let container = docker.run(Postgres::default());
5577 let port = container.get_host_port_ipv4(5432);
5578 let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5579 sleep(Duration::from_millis(250)).await;
5580
5581 let storage = Arc::new(
5582 Storage::connect(StorageConfig {
5583 database_url: db_url,
5584 max_connections: 2,
5585 connect_timeout: Duration::from_secs(10),
5586 object_store: ObjectStoreConfig::InMemory,
5587 })
5588 .await
5589 .expect("storage connect"),
5590 );
5591 let auth = Arc::new(AuthHandle::disabled());
5592 let audit = Arc::new(Audit::new(storage.pool().clone()));
5593 let attestation_vault: Arc<dyn AttestationVault> =
5594 Arc::new(InMemoryAttestationVault::new());
5595 let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5596 let service = RuntimeApiService::new(
5597 storage,
5598 auth,
5599 audit,
5600 attestation_vault,
5601 transparency,
5602 false,
5603 TransparencyScope::Disabled,
5604 );
5605
5606 let step_id = Uuid::new_v4();
5607 let dag = json!({
5608 "steps": [
5609 {
5610 "id": step_id,
5611 "type": "tool",
5612 "inputs": {
5613 "command": ["echo", "ok"]
5614 }
5615 }
5616 ]
5617 });
5618
5619 let mut request = build_request(dag);
5620 if let Some(spec) = request.spec.as_mut() {
5621 spec.seed = 0;
5622 }
5623
5624 let response = service.submit_run(Request::new(request)).await;
5625 assert!(response.is_err(), "expected seed validation failure");
5626 let status = response.err().unwrap();
5627 assert_eq!(status.code(), tonic::Code::InvalidArgument);
5628 assert!(status.message().contains("seed must be a positive integer"));
5629
5630 drop(container);
5631 }
5632
5633 #[tokio::test]
5634 async fn submit_run_invalid_dag_variants() {
5635 if std::fs::metadata("/var/run/docker.sock").is_err()
5636 && std::env::var("DOCKER_HOST").is_err()
5637 {
5638 eprintln!("Skipping API tests because Docker is unavailable");
5639 return;
5640 }
5641
5642 let docker = Cli::default();
5643 let container = docker.run(Postgres::default());
5644 let port = container.get_host_port_ipv4(5432);
5645 let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5646 sleep(Duration::from_millis(250)).await;
5647
5648 let storage = Arc::new(
5649 Storage::connect(StorageConfig {
5650 database_url: db_url,
5651 max_connections: 2,
5652 connect_timeout: Duration::from_secs(10),
5653 object_store: ObjectStoreConfig::InMemory,
5654 })
5655 .await
5656 .expect("storage connect"),
5657 );
5658 let auth = Arc::new(AuthHandle::disabled());
5659 let audit = Arc::new(Audit::new(storage.pool().clone()));
5660 let attestation_vault: Arc<dyn AttestationVault> =
5661 Arc::new(InMemoryAttestationVault::new());
5662 let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5663 let service = RuntimeApiService::new(
5664 Arc::clone(&storage),
5665 Arc::clone(&auth),
5666 Arc::clone(&audit),
5667 Arc::clone(&attestation_vault),
5668 transparency,
5669 false,
5670 TransparencyScope::Disabled,
5671 );
5672
5673 let step_a = Uuid::new_v4();
5674 let step_b = Uuid::new_v4();
5675 let valid_step = json!({
5676 "id": step_a,
5677 "type": "tool",
5678 "inputs": {}
5679 });
5680
5681 let cases: Vec<(&str, serde_json::Value, &str)> = vec![
5682 (
5683 "missing_steps",
5684 json!({ "edges": [] }),
5685 "dag_json.steps must be an array",
5686 ),
5687 (
5688 "unknown_step_type",
5689 json!({ "steps": [ { "id": step_a, "type": "unknown", "inputs": {} } ] }),
5690 "step schema validation failed",
5691 ),
5692 (
5693 "duplicate_step_id",
5694 json!({ "steps": [valid_step.clone(), valid_step.clone()] }),
5695 "duplicate step.id",
5696 ),
5697 (
5698 "self_edge",
5699 json!({ "steps": [valid_step.clone()], "edges": [[step_a, step_a]] }),
5700 "edge cannot reference the same step twice",
5701 ),
5702 (
5703 "edge_unknown_target",
5704 json!({ "steps": [valid_step.clone()], "edges": [[step_a, step_b]] }),
5705 "edge references unknown step id",
5706 ),
5707 (
5708 "duplicate_edge",
5709 json!({ "steps": [valid_step.clone(), {"id": step_b, "type": "tool", "inputs": {}}], "edges": [[step_a, step_b], [step_a, step_b]] }),
5710 "duplicate edge detected",
5711 ),
5712 ];
5713
5714 for (label, dag, expected) in cases {
5715 let request = build_request(dag);
5716 let response = service.submit_run(Request::new(request)).await;
5717 assert!(response.is_err(), "case {label} unexpectedly succeeded");
5718 let status = response.err().unwrap();
5719 assert_eq!(status.code(), tonic::Code::InvalidArgument, "case {label}");
5720 assert!(
5721 status.message().contains(expected),
5722 "case {label} expected '{expected}' but got '{}'",
5723 status.message()
5724 );
5725 }
5726
5727 drop(container);
5728 }
5729
5730 #[tokio::test]
5731 async fn submit_run_accepts_valid_dag_and_persists_steps() {
5732 if std::fs::metadata("/var/run/docker.sock").is_err()
5733 && std::env::var("DOCKER_HOST").is_err()
5734 {
5735 eprintln!("Skipping API tests because Docker is unavailable");
5736 return;
5737 }
5738
5739 let docker = Cli::default();
5740 let container = docker.run(Postgres::default());
5741 let port = container.get_host_port_ipv4(5432);
5742 let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5743 sleep(Duration::from_millis(250)).await;
5744
5745 let storage = Arc::new(
5746 Storage::connect(StorageConfig {
5747 database_url: db_url,
5748 max_connections: 2,
5749 connect_timeout: Duration::from_secs(10),
5750 object_store: ObjectStoreConfig::InMemory,
5751 })
5752 .await
5753 .expect("storage connect"),
5754 );
5755 let auth = Arc::new(AuthHandle::disabled());
5756 let audit = Arc::new(Audit::new(storage.pool().clone()));
5757 let attestation_vault: Arc<dyn AttestationVault> =
5758 Arc::new(InMemoryAttestationVault::new());
5759 let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5760 let service = RuntimeApiService::new(
5761 Arc::clone(&storage),
5762 Arc::clone(&auth),
5763 Arc::clone(&audit),
5764 Arc::clone(&attestation_vault),
5765 transparency,
5766 false,
5767 TransparencyScope::Disabled,
5768 );
5769
5770 let step_a = Uuid::new_v4();
5771 let step_b = Uuid::new_v4();
5772 let step_c = Uuid::new_v4();
5773
5774 let dag = serde_json::json!({
5775 "steps": [
5776 {
5777 "id": step_a,
5778 "type": "tool",
5779 "inputs": {},
5780 "policy": {"priority": 5}
5781 },
5782 {
5783 "id": step_b,
5784 "type": "llm",
5785 "inputs": {},
5786 "policy": {"priority": 3}
5787 },
5788 {
5789 "id": step_c,
5790 "type": "reduce",
5791 "inputs": {},
5792 "policy": {"priority": 1}
5793 }
5794 ],
5795 "edges": [
5796 [step_a, step_b],
5797 [step_b, step_c]
5798 ]
5799 });
5800
5801 let request = build_request(dag);
5802 let response = service
5803 .submit_run(Request::new(request))
5804 .await
5805 .expect("valid run should succeed")
5806 .into_inner();
5807
5808 let run_uuid = Uuid::parse_str(&response.run_id).expect("run_id must be UUID");
5809 let persisted_run = storage
5810 .fetch_run(run_uuid)
5811 .await
5812 .expect("fetch run")
5813 .expect("run should exist");
5814 assert_eq!(persisted_run.status, models::RunStatus::Pending);
5815
5816 let steps = storage
5817 .fetch_steps_for_run(run_uuid)
5818 .await
5819 .expect("fetch steps");
5820 assert_eq!(steps.len(), 3);
5821
5822 let mut by_idx = steps;
5823 by_idx.sort_by_key(|s| s.idx);
5824
5825 assert_eq!(by_idx[0].step_id, step_a);
5826 assert_eq!(by_idx[0].status, models::StepStatus::Queued);
5827 assert_eq!(by_idx[0].priority, 5);
5828 assert_eq!(by_idx[0].pending_dependencies, 0);
5829
5830 assert_eq!(by_idx[1].step_id, step_b);
5831 assert_eq!(by_idx[1].status, models::StepStatus::Blocked);
5832 assert_eq!(by_idx[1].priority, 3);
5833 assert_eq!(by_idx[1].pending_dependencies, 1);
5834
5835 assert_eq!(by_idx[2].step_id, step_c);
5836 assert_eq!(by_idx[2].status, models::StepStatus::Blocked);
5837 assert_eq!(by_idx[2].priority, 1);
5838 assert_eq!(by_idx[2].pending_dependencies, 1);
5839 assert!(
5840 by_idx
5841 .iter()
5842 .all(|s| s.total_dependencies >= s.pending_dependencies),
5843 "dependency counters should be consistent"
5844 );
5845 drop(container);
5846 }
5847
5848 #[tokio::test]
5849 async fn submit_run_respects_idempotency_key() {
5850 if std::fs::metadata("/var/run/docker.sock").is_err()
5851 && std::env::var("DOCKER_HOST").is_err()
5852 {
5853 eprintln!("Skipping API tests because Docker is unavailable");
5854 return;
5855 }
5856
5857 let docker = Cli::default();
5858 let container = docker.run(Postgres::default());
5859 let port = container.get_host_port_ipv4(5432);
5860 let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5861 sleep(Duration::from_millis(250)).await;
5862
5863 let storage = Arc::new(
5864 Storage::connect(StorageConfig {
5865 database_url: db_url,
5866 max_connections: 2,
5867 connect_timeout: Duration::from_secs(10),
5868 object_store: ObjectStoreConfig::InMemory,
5869 })
5870 .await
5871 .expect("storage connect"),
5872 );
5873 let auth = Arc::new(AuthHandle::disabled());
5874 let audit = Arc::new(Audit::new(storage.pool().clone()));
5875 let attestation_vault: Arc<dyn AttestationVault> =
5876 Arc::new(InMemoryAttestationVault::new());
5877 let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5878 let service = RuntimeApiService::new(
5879 Arc::clone(&storage),
5880 Arc::clone(&auth),
5881 Arc::clone(&audit),
5882 Arc::clone(&attestation_vault),
5883 transparency,
5884 false,
5885 TransparencyScope::Disabled,
5886 );
5887
5888 let step_id = Uuid::new_v4();
5889 let dag = serde_json::json!({
5890 "steps": [{
5891 "id": step_id,
5892 "type": "tool",
5893 "inputs": {}
5894 }]
5895 });
5896 let raw_key = " idempotent-key ";
5897 let normalized_key = "idempotent-key";
5898
5899 let first = service
5900 .submit_run(Request::new(build_request_with_key(dag.clone(), raw_key)))
5901 .await
5902 .expect("first submission")
5903 .into_inner();
5904
5905 let second = service
5906 .submit_run(Request::new(build_request_with_key(dag, raw_key)))
5907 .await
5908 .expect("second submission")
5909 .into_inner();
5910
5911 assert_eq!(
5912 first.run_id, second.run_id,
5913 "idempotent submissions should reuse run_id"
5914 );
5915
5916 let stored = storage
5917 .fetch_run_by_idempotency_key(normalized_key)
5918 .await
5919 .expect("lookup idempotent run")
5920 .expect("run should exist for idempotency key");
5921 assert_eq!(stored.run_id.to_string(), first.run_id);
5922
5923 drop(container);
5924 }
5925
5926 #[tokio::test]
5927 async fn get_attestations_returns_records() {
5928 if std::fs::metadata("/var/run/docker.sock").is_err()
5929 && std::env::var("DOCKER_HOST").is_err()
5930 {
5931 eprintln!("Skipping API tests because Docker is unavailable");
5932 return;
5933 }
5934
5935 let docker = Cli::default();
5936 let container = docker.run(Postgres::default());
5937 let port = container.get_host_port_ipv4(5432);
5938 let db_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
5939 sleep(Duration::from_millis(250)).await;
5940
5941 let storage = Arc::new(
5942 Storage::connect(StorageConfig {
5943 database_url: db_url,
5944 max_connections: 2,
5945 connect_timeout: Duration::from_secs(10),
5946 object_store: ObjectStoreConfig::InMemory,
5947 })
5948 .await
5949 .expect("storage connect"),
5950 );
5951 let auth = Arc::new(AuthHandle::disabled());
5952 let audit = Arc::new(Audit::new(storage.pool().clone()));
5953 let attestation_vault: Arc<dyn AttestationVault> =
5954 Arc::new(InMemoryAttestationVault::new());
5955 let transparency = transparency::local_transparency_log(Arc::clone(&storage));
5956 let service = RuntimeApiService::new(
5957 Arc::clone(&storage),
5958 Arc::clone(&auth),
5959 Arc::clone(&audit),
5960 Arc::clone(&attestation_vault),
5961 transparency,
5962 false,
5963 TransparencyScope::Disabled,
5964 );
5965
5966 let run_id = Uuid::new_v4();
5967 let attestation =
5968 Attestation::new(Uuid::new_v4(), Utc::now()).with_subject(TrustSubject::Run { run_id });
5969 attestation_vault
5970 .record(attestation.clone())
5971 .await
5972 .expect("record attestation");
5973
5974 let response = service
5975 .get_attestations(Request::new(runtime::GetAttestationsRequest {
5976 attestation_ids: vec![
5977 attestation.id.to_string(),
5978 attestation.id.to_string(),
5979 Uuid::new_v4().to_string(),
5980 ],
5981 }))
5982 .await
5983 .expect("get_attestations RPC")
5984 .into_inner();
5985
5986 assert_eq!(response.attestations.len(), 1);
5987 let returned = &response.attestations[0];
5988 assert_eq!(returned.attestation_id, attestation.id.to_string());
5989 match returned
5990 .subject
5991 .as_ref()
5992 .and_then(|subject| subject.kind.as_ref())
5993 {
5994 Some(runtime::attestation_subject::Kind::Run(run_subject)) => {
5995 assert_eq!(run_subject.run_id, run_id.to_string());
5996 }
5997 other => panic!("unexpected subject {other:?}"),
5998 }
5999
6000 drop(container);
6001 }
6002
6003 #[test]
6004 fn compute_drift_reports_within_tolerance() {
6005 let recorded = json!({
6006 "tokens": 10,
6007 "nested": { "value": "ok" }
6008 });
6009 let current = recorded.clone();
6010
6011 let result = compute_drift(
6012 &recorded,
6013 ¤t,
6014 DEFAULT_TOKEN_TOLERANCE,
6015 DEFAULT_COST_TOLERANCE,
6016 );
6017 assert!(result.within_tolerance);
6018 assert_eq!(result.token_delta, Some(0.0));
6019 assert_eq!(result.cost_delta, None);
6020 }
6021
6022 #[test]
6023 fn compute_drift_flags_token_delta() {
6024 let recorded = json!({
6025 "tokens": 10,
6026 "cost": 1.0,
6027 "payload": { "message": "hello" }
6028 });
6029 let current = json!({
6030 "tokens": 15,
6031 "cost": 1.0,
6032 "payload": { "message": "hello" }
6033 });
6034
6035 let result = compute_drift(
6036 &recorded,
6037 ¤t,
6038 DEFAULT_TOKEN_TOLERANCE,
6039 DEFAULT_COST_TOLERANCE,
6040 );
6041 assert!(!result.within_tolerance);
6042 assert_eq!(result.token_delta, Some(5.0));
6043 assert_eq!(result.cost_delta, Some(0.0));
6044 }
6045}