Skip to main content

fleetforge_runtime/
lib.rs

1//! FleetForge runtime state machine and orchestrator components.
2
3pub mod adapters;
4pub mod aibom;
5mod capability;
6pub mod executor;
7pub mod features;
8pub mod gateway;
9pub mod guardrails;
10pub mod memory;
11pub mod model;
12pub mod otel;
13pub mod policy;
14pub mod replay;
15pub mod scheduler;
16
17use std::{env, sync::Arc};
18
19use anyhow::{anyhow, Result};
20use chrono::{Duration as ChronoDuration, Utc};
21use fleetforge_bus::{run_outbox_forwarder_with_exporters, BusConfig};
22use fleetforge_common::licensing::{feature_allowed, LicensedFeature};
23use fleetforge_runtime_api::{start_api_server, RuntimeMetadata};
24use fleetforge_storage::retention::{
25    delete_expired as purge_retention, RetentionConfig as StorageRetentionConfig,
26};
27use fleetforge_storage::Storage;
28use fleetforge_telemetry::{audit::Audit, TelemetryGuard};
29use serde_json::json;
30use tracing::{info, warn};
31
32pub use adapters::{AgentRunAction, AgentRunAdapter, AgentRunContext, AgentRunEnvelope};
33#[cfg(feature = "insecure-shell")]
34pub use executor::ShellExecutor;
35pub use executor::{
36    BudgetCtx, DeterministicReplayExecutor, DockerToolExecutor, ExecutorRegistry,
37    FirecrackerToolExecutor, HttpProxyExecutor, LangGraphAgentExecutor, LlmExecutor, ResourceUsage,
38    StepCtx, StepExecutionResult, StepExecutor, ToolRouterExecutor,
39};
40pub use features::{set_trust_mesh_alpha, trust_mesh_alpha_enabled};
41pub use fleetforge_common::schemas::{
42    AGENT_RUN_SCHEMA_JSON, ARTIFACT_SCHEMA_JSON, RUN_EVENT_SCHEMA_JSON, RUN_SCHEMA_JSON,
43    RUN_SPEC_SCHEMA_JSON, STEP_SCHEMA_JSON,
44};
45pub use fleetforge_common::validation::{
46    validate_agent_run, validate_artifact, validate_run, validate_run_event, validate_run_spec,
47    validate_step_spec, SchemaErrorDetail, SchemaValidationError, StepSpecError,
48    StepSpecErrorDetail,
49};
50pub use fleetforge_policy::{
51    shared_default_pack, AllowAllPolicy, BasicPiiPolicy, Decision as PolicyDecision, PolicyEngine,
52    PolicyRequest, RegulatedPack, RegulatedVertical,
53};
54pub use fleetforge_prompt::{
55    CompiledPrompt, ModelResponse, ModelUsage, PromptCompiler, PromptRegistry,
56};
57pub use fleetforge_trust::{
58    digest_bytes, digest_json, Attestation, AttestationVault, InMemoryAttestationVault,
59    ObjectStoreAttestationVault, Trust, TrustBoundary, TrustDecision, TrustOrigin, TrustSource,
60    TrustSubject, TrustVerdict, Trusted, Untrusted, TRUST_MESH_ALPHA_FLAG,
61};
62pub use gateway::speech::{NoopStt, NoopTts, SpeechToText, TextToSpeech};
63pub use gateway::{
64    anthropic::{AnthropicConfig, AnthropicLanguageModel},
65    openai::{OpenAiConfig, OpenAiLanguageModel},
66    GatewayRegistry, LanguageModel, ToolAdapter,
67};
68pub use guardrails::{OutputGuardPolicy, PolicyBundle, PromptInjectionFallbackPolicy};
69pub use memory::{InMemoryAdapter, MemoryAdapter, MemoryRecord, VectorMatch, VectorMemoryAdapter};
70pub use model::{
71    ChatMessage, ChatRole, ContextSource, LlmInputs, QueuedStep, ResponseSchema, RunId, RunStatus,
72    StepId, StepSpec, StepStatus, StepType, ToolChoice, ToolSpec,
73};
74pub use policy::{enforce_policy, PolicyOutcome, RuntimePolicyPack};
75pub use replay::{
76    DriftTolerance, ReplayExecutor, ReplayOutcome, ReplayStrategy, StepDrift, StepReplay,
77};
78pub use scheduler::{Scheduler, SchedulerConfig};
79
80#[cfg(test)]
81mod memory_test;
82
83#[cfg(test)]
84mod e2e {
85    use super::*;
86    use crate::replay::{DriftTolerance, ReplayExecutor, ReplayStrategy};
87    use anyhow::{anyhow, Context, Result as AnyResult};
88    use base64::engine::general_purpose::STANDARD as BASE64;
89    use base64::Engine;
90    use chrono::Utc;
91    use clickhouse::Client;
92    use fleetforge_common::prost_json::{json_to_prost_struct, prost_struct_to_json};
93    use fleetforge_contracts::runtime::{
94        runtime_service_client::RuntimeServiceClient, GetRunRequest, RunSpec, SubmitRunRequest,
95    };
96    use fleetforge_contracts::tap::{
97        self, subscribe_run_request, tap_service_client::TapServiceClient, RunEventEnvelope,
98        SubscribeRunAck, SubscribeRunOpen, SubscribeRunRequest,
99    };
100    use fleetforge_policy::AllowAllPolicy;
101    use fleetforge_runtime_api::RuntimeMetadata;
102    use fleetforge_storage::{models, object_store::ObjectStoreConfig, StorageConfig};
103    use fleetforge_telemetry::audit::Audit;
104    use fleetforge_telemetry::otel::{self as telemetry_otel, Value as OtelValue};
105    use fleetforge_trust::{verify_capability_token, CapabilityToken, TrustSubject};
106    use hmac::{Hmac, Mac};
107    use object_store::path::Path;
108    use opentelemetry_sdk::export::trace::{ExportResult, SpanData, SpanExporter};
109    use opentelemetry_sdk::trace;
110    use reqwest::StatusCode;
111    use serde_json::{json, Value};
112    use sha2::Sha256;
113    use sqlx::Row;
114    use std::{
115        collections::{BTreeMap, HashMap, HashSet},
116        fs,
117        net::TcpListener,
118        path::PathBuf,
119        sync::{Arc, Mutex},
120        time::Duration,
121    };
122    use testcontainers::clients::Cli;
123    use testcontainers::core::WaitFor;
124    use testcontainers::images::generic::GenericImage;
125    use testcontainers::images::postgres::Postgres;
126    use tokio::sync::mpsc;
127    use tokio::time::sleep;
128    use tokio_stream::wrappers::ReceiverStream;
129    use tonic::metadata::MetadataValue;
130    use tracing::subscriber::DefaultGuard;
131    use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry};
132    use uuid::Uuid;
133
134    struct EchoToolExecutor;
135
136    #[async_trait::async_trait]
137    impl StepExecutor for EchoToolExecutor {
138        fn kind(&self) -> &'static str {
139            "tool"
140        }
141
142        async fn execute(&self, ctx: &StepCtx) -> anyhow::Result<StepExecutionResult> {
143            let command = ctx
144                .untrusted_inputs
145                .as_ref()
146                .get("command")
147                .and_then(Value::as_array)
148                .ok_or_else(|| anyhow!("missing command array"))?;
149            let message = command
150                .iter()
151                .skip(1)
152                .filter_map(Value::as_str)
153                .collect::<Vec<_>>()
154                .join(" ");
155            let stdout = format!("{}\n", message);
156            let output = json!({
157                "stdout": stdout,
158                "stderr": "",
159                "exit_code": 0,
160            });
161
162            Ok(StepExecutionResult::with_usage(
163                output,
164                ResourceUsage {
165                    tokens: Some(100),
166                    cost: Some(0.001),
167                },
168            ))
169        }
170    }
171
172    fn test_runtime_metadata() -> Arc<RuntimeMetadata> {
173        Arc::new(RuntimeMetadata::new(
174            "test-sha",
175            "fleetforge.policy.default",
176            "test-digest",
177            true,
178        ))
179    }
180
181    fn repo_root() -> PathBuf {
182        PathBuf::from(env!("CARGO_MANIFEST_DIR"))
183            .join("..")
184            .join("..")
185    }
186
187    fn load_agent_team_spec() -> AnyResult<(
188        String,
189        Option<prost_types::Struct>,
190        i64,
191        HashMap<String, String>,
192    )> {
193        let pack_dir = repo_root().join("examples/_packs/demo-pack/agent_team");
194        let spec_value: Value =
195            serde_json::from_str(&fs::read_to_string(pack_dir.join("run_spec.json"))?)?;
196
197        let dag_value = match spec_value.get("dag_json").cloned().unwrap_or(Value::Null) {
198            Value::String(path) if path.starts_with("@file:") => {
199                let dag_path = pack_dir.join(path.trim_start_matches("@file:"));
200                serde_json::from_str(&fs::read_to_string(dag_path)?)?
201            }
202            other => other,
203        };
204
205        let dag_json = serde_json::to_string(&dag_value)?;
206
207        let inputs_struct = match spec_value.get("inputs") {
208            Some(Value::Null) | None => None,
209            Some(value) => Some(json_to_prost_struct(value)?),
210        };
211
212        let seed = spec_value
213            .get("seed")
214            .and_then(Value::as_i64)
215            .ok_or_else(|| anyhow!("run_spec.json must include an explicit integer seed"))?;
216        if seed <= 0 {
217            return Err(anyhow!("run_spec.json seed must be a positive integer"));
218        }
219
220        let labels = spec_value
221            .get("labels")
222            .and_then(Value::as_object)
223            .map(|map| {
224                map.iter()
225                    .filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
226                    .collect::<HashMap<_, _>>()
227            })
228            .unwrap_or_default();
229
230        Ok((dag_json, inputs_struct, seed, labels))
231    }
232
233    fn reserve_port() -> u16 {
234        TcpListener::bind("127.0.0.1:0")
235            .expect("bind local port")
236            .local_addr()
237            .unwrap()
238            .port()
239    }
240
241    fn docker_available() -> bool {
242        std::env::var("DOCKER_HOST").is_ok() || fs::metadata("/var/run/docker.sock").is_ok()
243    }
244
245    async fn wait_for_clickhouse(client: &Client) -> AnyResult<()> {
246        const RETRIES: usize = 10;
247        for attempt in 0..RETRIES {
248            match client.query("SELECT 1").fetch_one::<(u64,)>().await {
249                Ok(_) => return Ok(()),
250                Err(err) if attempt + 1 < RETRIES => {
251                    tokio::time::sleep(Duration::from_millis(200)).await;
252                    eprintln!("waiting for ClickHouse readiness: {err}");
253                }
254                Err(err) => return Err(err.into()),
255            }
256        }
257        Ok(())
258    }
259
260    async fn wait_for_run(
261        client: &mut RuntimeServiceClient<tonic::transport::Channel>,
262        run_id: &str,
263    ) -> AnyResult<runtime::RunDetail> {
264        for _ in 0..40 {
265            let mut request = tonic::Request::new(GetRunRequest {
266                run_id: run_id.to_string(),
267            });
268            request.metadata_mut().insert(
269                "authorization",
270                MetadataValue::try_from("Bearer demo-reader")?,
271            );
272            let response = client.get_run(request).await?.into_inner();
273            if let Some(run) = response.run {
274                if run.status == runtime::RunStatus::RunStatusSucceeded as i32 {
275                    return Ok(run);
276                }
277            }
278            sleep(Duration::from_millis(250)).await;
279        }
280        Err(anyhow!("run did not succeed in time"))
281    }
282
283    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
284    async fn golden_path_api_surface_acceptance() -> AnyResult<()> {
285        if !docker_available() {
286            eprintln!("Skipping API surface acceptance test because Docker is unavailable");
287            return Ok(());
288        }
289
290        let docker = Cli::default();
291        let postgres = docker.run(Postgres::default());
292        let pg_port = postgres.get_host_port_ipv4(5432);
293        let database_url = format!("postgres://postgres:postgres@127.0.0.1:{pg_port}/postgres");
294        sleep(Duration::from_millis(250)).await;
295
296        let clickhouse_image = GenericImage::new("clickhouse/clickhouse-server", "23.11")
297            .with_wait_for(WaitFor::message("Ready for connections"));
298        let clickhouse = docker.run(clickhouse_image);
299        let ch_port = clickhouse.get_host_port_ipv4(8123);
300        let ch_url = format!("http://127.0.0.1:{ch_port}");
301
302        let admin_client = Client::default()
303            .with_url(ch_url.clone())
304            .with_database("default");
305        wait_for_clickhouse(&admin_client).await?;
306        admin_client
307            .command("CREATE DATABASE IF NOT EXISTS telemetry")
308            .execute()
309            .await?;
310        admin_client
311            .command(
312                "CREATE TABLE IF NOT EXISTS telemetry.run_events (\
313                    run_id UUID, \
314                    created_at DateTime64(3, 'UTC'), \
315                    status String, \
316                    attributes JSON\
317                ) ENGINE=MergeTree ORDER BY (created_at, run_id)",
318            )
319            .execute()
320            .await?;
321        admin_client
322            .command(
323                "CREATE TABLE IF NOT EXISTS telemetry.step_events (\
324                    run_id UUID, \
325                    step_id UUID, \
326                    status String, \
327                    attempt Int32, \
328                    started_at DateTime64(3, 'UTC'), \
329                    finished_at Nullable(DateTime64(3, 'UTC')), \
330                    attributes JSON\
331                ) ENGINE=MergeTree ORDER BY (started_at, run_id, step_id)",
332            )
333            .execute()
334            .await?;
335
336        let storage = Arc::new(
337            Storage::connect(StorageConfig {
338                database_url,
339                max_connections: 4,
340                connect_timeout: Duration::from_secs(10),
341                object_store: ObjectStoreConfig::InMemory,
342            })
343            .await?,
344        );
345        let audit = Arc::new(Audit::new(storage.pool().clone()));
346        let attestation_vault: Arc<dyn AttestationVault> = Arc::new(
347            ObjectStoreAttestationVault::new(storage.object_store(), storage.pool().clone()),
348        );
349
350        let mut registry = ExecutorRegistry::new();
351        registry.register(Arc::new(EchoToolExecutor))?;
352
353        let scheduler = Arc::new(Scheduler::new(
354            Arc::clone(&storage),
355            Arc::new(registry),
356            AllowAllPolicy::shared(),
357            Arc::clone(&audit),
358            Arc::clone(&attestation_vault),
359            SchedulerConfig::default(),
360        ));
361
362        struct TaskAbortOnDrop(tokio::task::JoinHandle<()>);
363        impl TaskAbortOnDrop {
364            fn abort(&mut self) {
365                self.0.abort();
366            }
367        }
368        impl Drop for TaskAbortOnDrop {
369            fn drop(&mut self) {
370                self.0.abort();
371            }
372        }
373
374        let scheduler_handle = tokio::spawn({
375            let scheduler = Arc::clone(&scheduler);
376            async move {
377                scheduler.run().await.expect("scheduler loop");
378            }
379        });
380        let mut scheduler_handle = TaskAbortOnDrop(scheduler_handle);
381
382        let grpc_port = reserve_port();
383        let http_port = reserve_port();
384        std::env::set_var(
385            "FLEETFORGE_API_TOKENS",
386            "writer=demo-writer,reader=demo-reader",
387        );
388        std::env::set_var("FLEETFORGE_API_ADDR", format!("127.0.0.1:{grpc_port}"));
389        std::env::set_var("FLEETFORGE_HTTP_ADDR", format!("127.0.0.1:{http_port}"));
390        std::env::set_var("CLICKHOUSE_DSN", ch_url.clone());
391        std::env::set_var("CLICKHOUSE_DATABASE", "telemetry");
392        std::env::set_var("CLICKHOUSE_RUN_TABLE", "run_events");
393        std::env::set_var("CLICKHOUSE_STEP_TABLE", "step_events");
394
395        struct EnvCleanup;
396        impl Drop for EnvCleanup {
397            fn drop(&mut self) {
398                for key in [
399                    "FLEETFORGE_API_TOKENS",
400                    "FLEETFORGE_API_ADDR",
401                    "FLEETFORGE_HTTP_ADDR",
402                    "CLICKHOUSE_DSN",
403                    "CLICKHOUSE_DATABASE",
404                    "CLICKHOUSE_RUN_TABLE",
405                    "CLICKHOUSE_STEP_TABLE",
406                ] {
407                    std::env::remove_var(key);
408                }
409            }
410        }
411        let _env_guard = EnvCleanup;
412
413        let api_storage = Arc::clone(&storage);
414        let api_audit = Arc::clone(&audit);
415        let version_meta = test_runtime_metadata();
416        let api_handle = tokio::spawn(async move {
417            if let Err(err) = start_api_server(api_storage, api_audit, version_meta).await {
418                eprintln!("API server exited: {err:?}");
419            }
420        });
421        let mut api_handle = TaskAbortOnDrop(api_handle);
422
423        sleep(Duration::from_millis(500)).await;
424
425        let dag_step_id = "11111111-1111-4111-8111-111111111111";
426        let dag_value = json!({
427            "steps": [{
428                "id": dag_step_id,
429                "type": "tool",
430                "inputs": {
431                    "command": ["echo", "hello", "acceptance"]
432                }
433            }],
434            "edges": []
435        });
436        let dag_json_string = serde_json::to_string(&dag_value)?;
437
438        let http_client = reqwest::Client::new();
439        let submit_body = json!({
440            "dag_json": dag_value,
441            "inputs": {},
442            "seed": 42,
443            "labels": { "purpose": "acceptance" }
444        });
445
446        let submit_response = http_client
447            .post(format!("http://127.0.0.1:{http_port}/v1/runs"))
448            .header("Authorization", "Bearer demo-writer")
449            .json(&submit_body)
450            .send()
451            .await?;
452        assert_eq!(submit_response.status(), StatusCode::CREATED);
453        let submit_json: Value = submit_response.json().await?;
454        let run_id_http = submit_json["run_id"]
455            .as_str()
456            .ok_or_else(|| anyhow!("missing run_id in HTTP response"))?
457            .to_string();
458
459        let detail_response = http_client
460            .get(format!(
461                "http://127.0.0.1:{http_port}/v1/runs/{run_id_http}"
462            ))
463            .header("Authorization", "Bearer demo-reader")
464            .send()
465            .await?;
466        assert_eq!(detail_response.status(), StatusCode::OK);
467        let detail_json: Value = detail_response.json().await?;
468        assert_eq!(
469            detail_json["seed"].as_i64(),
470            Some(42),
471            "HTTP readback should preserve seed"
472        );
473
474        let endpoint = format!("http://127.0.0.1:{grpc_port}");
475        let mut runtime_client = RuntimeServiceClient::connect(endpoint.clone()).await?;
476        let run_detail = wait_for_run(&mut runtime_client, &run_id_http).await?;
477        assert_eq!(
478            run_detail.seed, 42,
479            "gRPC GetRun should return the same seed"
480        );
481        let run_uuid = Uuid::parse_str(&run_id_http)?;
482        let steps = storage
483            .fetch_steps_for_run(run_uuid)
484            .await
485            .context("fetch steps for run")?;
486        let first_step = steps.first().context("expected recorded step")?;
487
488        let mut tap_client = TapServiceClient::connect(endpoint.clone()).await?;
489        let (mut tx, rx) = mpsc::channel::<SubscribeRunRequest>(32);
490        tx.send(SubscribeRunRequest {
491            request: Some(subscribe_run_request::Request::Open(SubscribeRunOpen {
492                run_ids: vec![run_id_http.clone()],
493                step_ids: Vec::new(),
494                kinds: Vec::new(),
495                severities: Vec::new(),
496                tags: Vec::new(),
497                since_offset: 0,
498                batch_size: 128,
499                max_inflight: 64,
500                heartbeat_interval_ms: 1_000,
501                cursor: "".into(),
502            })),
503        })
504        .await?;
505        let outbound = ReceiverStream::new(rx);
506        let mut tap_request = tonic::Request::new(outbound);
507        tap_request.metadata_mut().insert(
508            "authorization",
509            MetadataValue::try_from("Bearer demo-reader")?,
510        );
511        let mut stream = tap_client.subscribe_run(tap_request).await?.into_inner();
512        let mut saw_event = false;
513        while let Some(mut response) = stream.message().await? {
514            if response.heartbeat {
515                continue;
516            }
517            saw_event = true;
518            let offset = response.offset;
519            if offset > 0 {
520                tx.send(SubscribeRunRequest {
521                    request: Some(subscribe_run_request::Request::Ack(SubscribeRunAck {
522                        offset,
523                    })),
524                })
525                .await?;
526            }
527            break;
528        }
529        drop(tx);
530        assert!(saw_event, "expected tap stream to yield events");
531
532        #[derive(Clone, serde::Serialize, clickhouse::Row)]
533        struct RunEventInsert {
534            run_id: Uuid,
535            created_at: chrono::DateTime<Utc>,
536            status: String,
537            attributes: Value,
538        }
539
540        #[derive(Clone, serde::Serialize, clickhouse::Row)]
541        struct StepEventInsert {
542            run_id: Uuid,
543            step_id: Uuid,
544            status: String,
545            attempt: i32,
546            started_at: chrono::DateTime<Utc>,
547            finished_at: Option<chrono::DateTime<Utc>>,
548            attributes: Value,
549        }
550
551        let telemetry_client = Client::default()
552            .with_url(ch_url.clone())
553            .with_database("telemetry");
554        let now = Utc::now();
555
556        let mut run_insert = telemetry_client.insert("telemetry.run_events")?;
557        run_insert
558            .write(&RunEventInsert {
559                run_id: run_uuid,
560                created_at: now,
561                status: "succeeded".to_string(),
562                attributes: json!({ "source": "acceptance" }),
563            })
564            .await?;
565        run_insert.end().await?;
566
567        let mut step_insert = telemetry_client.insert("telemetry.step_events")?;
568        step_insert
569            .write(&StepEventInsert {
570                run_id: run_uuid,
571                step_id: first_step.step_id,
572                status: "succeeded".to_string(),
573                attempt: first_step.attempt,
574                started_at: first_step.created_at,
575                finished_at: Some(Utc::now()),
576                attributes: json!({ "source": "acceptance" }),
577            })
578            .await?;
579        step_insert.end().await?;
580
581        let mut tap_client = TapServiceClient::connect(endpoint.clone()).await?;
582        let mut query_request = tap::QueryEventsRequest {
583            run_ids: vec![run_id_http.clone()],
584            workspace_ids: Vec::new(),
585            kinds: Vec::new(),
586            policy_effects: Vec::new(),
587            start_time: None,
588            end_time: None,
589            page_size: 10,
590            page_token: String::new(),
591            source: tap::QueryEventSource::QueryEventSourceRun as i32,
592            step_ids: Vec::new(),
593            ascending: true,
594            severities: Vec::new(),
595        };
596        let mut query_req = tonic::Request::new(query_request);
597        query_req.metadata_mut().insert(
598            "authorization",
599            MetadataValue::try_from("Bearer demo-reader")?,
600        );
601        let query_response = tap_client.query_events(query_req).await?.into_inner();
602        assert!(
603            !query_response.events.is_empty(),
604            "expected ClickHouse-backed QueryEvents response"
605        );
606
607        let mut labels = HashMap::new();
608        labels.insert("purpose".to_string(), "breakpoint".to_string());
609        let run2_spec = RunSpec {
610            dag_json: dag_json_string.clone(),
611            inputs: None,
612            seed: 99,
613            labels,
614            breakpoints: vec![dag_step_id.to_string()],
615            ..Default::default()
616        };
617
618        let mut submit_req = tonic::Request::new(SubmitRunRequest {
619            spec: Some(run2_spec),
620            idempotency_key: String::new(),
621        });
622        submit_req.metadata_mut().insert(
623            "authorization",
624            MetadataValue::try_from("Bearer demo-writer")?,
625        );
626        let run2_id = runtime_client
627            .submit_run(submit_req)
628            .await?
629            .into_inner()
630            .run_id;
631
632        let mut paused = false;
633        for _ in 0..40 {
634            let mut get_req = tonic::Request::new(GetRunRequest {
635                run_id: run2_id.clone(),
636            });
637            get_req.metadata_mut().insert(
638                "authorization",
639                MetadataValue::try_from("Bearer demo-reader")?,
640            );
641            let response = runtime_client.get_run(get_req).await?.into_inner();
642            if let Some(run) = response.run {
643                if run.status == runtime::RunStatus::RunStatusPausedAtStep as i32 {
644                    paused = true;
645                    break;
646                }
647            }
648            sleep(Duration::from_millis(250)).await;
649        }
650        assert!(paused, "run should pause at declared breakpoint");
651
652        let mut update_req = tonic::Request::new(runtime::UpdateBreakpointsRequest {
653            run_id: run2_id.clone(),
654            add: Vec::new(),
655            remove: Vec::new(),
656            clear_all: true,
657        });
658        update_req.metadata_mut().insert(
659            "authorization",
660            MetadataValue::try_from("Bearer demo-writer")?,
661        );
662        let update_response = runtime_client
663            .update_breakpoints(update_req)
664            .await?
665            .into_inner();
666        assert!(
667            update_response.breakpoints.is_empty(),
668            "clear_all should remove pending breakpoints"
669        );
670
671        let mut resume_req = tonic::Request::new(runtime::ResumeRunRequest {
672            run_id: run2_id.clone(),
673        });
674        resume_req.metadata_mut().insert(
675            "authorization",
676            MetadataValue::try_from("Bearer demo-writer")?,
677        );
678        let resume_response = runtime_client.resume_run(resume_req).await?.into_inner();
679        assert_eq!(
680            resume_response.status,
681            runtime::RunStatus::RunStatusRunning as i32,
682            "resume should transition run back to running"
683        );
684
685        let _ = wait_for_run(&mut runtime_client, &run2_id).await?;
686
687        scheduler_handle.abort();
688        api_handle.abort();
689        drop(clickhouse);
690        drop(postgres);
691
692        Ok(())
693    }
694
695    async fn submit_and_wait(
696        client: &mut RuntimeServiceClient<tonic::transport::Channel>,
697        tap_endpoint: &str,
698        spec: RunSpec,
699    ) -> AnyResult<(String, Vec<RunEventEnvelope>, runtime::RunDetail)> {
700        let submit_request = SubmitRunRequest {
701            spec: Some(spec),
702            idempotency_key: String::new(),
703        };
704
705        let mut request = tonic::Request::new(submit_request);
706        request.metadata_mut().insert(
707            "authorization",
708            MetadataValue::try_from("Bearer demo-writer")?,
709        );
710
711        let response = client.submit_run(request).await?.into_inner();
712        let run_id = response.run_id;
713
714        let mut tap_client = TapServiceClient::connect(tap_endpoint.to_string()).await?;
715        let (mut tx, rx) = mpsc::channel::<SubscribeRunRequest>(32);
716        tx.send(SubscribeRunRequest {
717            request: Some(subscribe_run_request::Request::Open(SubscribeRunOpen {
718                run_ids: vec![run_id.clone()],
719                step_ids: Vec::new(),
720                kinds: Vec::new(),
721                severities: Vec::new(),
722                tags: Vec::new(),
723                since_offset: 0,
724                batch_size: 256,
725                max_inflight: 64,
726                heartbeat_interval_ms: 1_000,
727            })),
728        })
729        .await?;
730
731        let outbound = ReceiverStream::new(rx);
732        let mut tap_request = tonic::Request::new(outbound);
733        tap_request.metadata_mut().insert(
734            "authorization",
735            MetadataValue::try_from("Bearer demo-reader")?,
736        );
737
738        let mut stream = tap_client.subscribe_run(tap_request).await?.into_inner();
739        let mut events = Vec::new();
740
741        while let Some(mut response) = stream.message().await? {
742            if response.heartbeat {
743                continue;
744            }
745
746            if let Some(event) = response.event.take() {
747                if response.offset > 0 {
748                    let ack = SubscribeRunRequest {
749                        request: Some(subscribe_run_request::Request::Ack(SubscribeRunAck {
750                            offset: response.offset,
751                        })),
752                    };
753                    if tx.send(ack).await.is_err() {
754                        break;
755                    }
756                }
757
758                let done = event.tags.iter().any(|tag| tag == "kind:run_succeeded");
759                events.push(event);
760                if done {
761                    break;
762                }
763            }
764        }
765
766        let run_detail = wait_for_run(client, &run_id).await?;
767        Ok((run_id, events, run_detail))
768    }
769
770    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
771    async fn golden_path_agent_team_replay_within_one_percent() -> AnyResult<()> {
772        if std::fs::metadata("/var/run/docker.sock").is_err()
773            && std::env::var("DOCKER_HOST").is_err()
774        {
775            eprintln!("Skipping E2E test because Docker is unavailable");
776            return Ok(());
777        }
778
779        let docker = Cli::default();
780        let container = docker.run(Postgres::default());
781        let port = container.get_host_port_ipv4(5432);
782        let database_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
783        sleep(Duration::from_millis(250)).await;
784
785        let storage = Arc::new(
786            Storage::connect(StorageConfig {
787                database_url,
788                max_connections: 4,
789                connect_timeout: Duration::from_secs(10),
790                object_store: ObjectStoreConfig::InMemory,
791            })
792            .await?,
793        );
794        let audit = Arc::new(Audit::new(storage.pool().clone()));
795
796        let mut registry = ExecutorRegistry::new();
797        registry.register(Arc::new(EchoToolExecutor))?;
798
799        let scheduler = Arc::new(Scheduler::new(
800            Arc::clone(&storage),
801            Arc::new(registry),
802            AllowAllPolicy::shared(),
803            Arc::clone(&audit),
804            Arc::clone(&attestation_vault),
805            SchedulerConfig::default(),
806        ));
807
808        let scheduler_handle = tokio::spawn({
809            let scheduler = Arc::clone(&scheduler);
810            async move {
811                scheduler.run().await.expect("scheduler loop");
812            }
813        });
814
815        let grpc_port = reserve_port();
816        let http_port = reserve_port();
817        std::env::set_var(
818            "FLEETFORGE_API_TOKENS",
819            "writer=demo-writer,reader=demo-reader",
820        );
821        std::env::set_var("FLEETFORGE_API_ADDR", format!("127.0.0.1:{grpc_port}"));
822        std::env::set_var("FLEETFORGE_HTTP_ADDR", format!("127.0.0.1:{http_port}"));
823
824        let api_storage = Arc::clone(&storage);
825        let api_audit = Arc::clone(&audit);
826        let version_meta = test_runtime_metadata();
827        let api_handle = tokio::spawn(async move {
828            if let Err(err) = start_api_server(api_storage, api_audit, version_meta).await {
829                eprintln!("API server exited: {err:?}");
830            }
831        });
832
833        sleep(Duration::from_millis(500)).await;
834
835        let endpoint = format!("http://127.0.0.1:{grpc_port}");
836        let channel = tonic::transport::Channel::from_shared(endpoint.clone())?
837            .connect()
838            .await?;
839        let mut client = RuntimeServiceClient::new(channel);
840
841        let (dag_json, inputs_struct, seed, labels) = load_agent_team_spec()?;
842        let run_spec = RunSpec {
843            dag_json: dag_json.clone(),
844            inputs: inputs_struct.clone(),
845            seed,
846            labels: labels.clone(),
847        };
848
849        let (run_a, events_a, _) =
850            submit_and_wait(&mut client, &endpoint, run_spec.clone()).await?;
851        assert!(events_a
852            .iter()
853            .any(|event| event.tags.iter().any(|tag| tag == "kind:run_succeeded")));
854
855        let trace_ids: HashSet<_> = events_a
856            .iter()
857            .filter_map(|event| event.trace.as_ref())
858            .map(|trace| trace.trace_id.clone())
859            .filter(|id| !id.is_empty())
860            .collect();
861        assert_eq!(
862            trace_ids.len(),
863            1,
864            "expected a single trace_id across the agent run"
865        );
866
867        let mut saw_budget = false;
868        let mut saw_policy = false;
869        for event in &events_a {
870            if let Some(raw) = event.raw.as_ref() {
871                let raw_json = prost_struct_to_json(raw);
872                if raw_json
873                    .get("budget")
874                    .and_then(|value| value.as_object())
875                    .is_some()
876                {
877                    saw_budget = true;
878                }
879                if raw_json
880                    .get("policy_decisions")
881                    .and_then(|value| value.as_object())
882                    .is_some()
883                {
884                    saw_policy = true;
885                }
886            }
887        }
888        assert!(saw_budget, "expected budget telemetry in stream payloads");
889        assert!(
890            saw_policy,
891            "expected policy decision summaries in stream payloads"
892        );
893
894        let steps_a = storage
895            .fetch_steps_for_run(Uuid::parse_str(&run_a)?)
896            .await?;
897
898        let (run_b, events_b, _) = submit_and_wait(&mut client, &endpoint, run_spec).await?;
899        assert!(events_b
900            .iter()
901            .any(|event| event.tags.iter().any(|tag| tag == "kind:run_succeeded")));
902
903        let steps_b = storage
904            .fetch_steps_for_run(Uuid::parse_str(&run_b)?)
905            .await?;
906
907        assert_eq!(steps_a.len(), steps_b.len(), "step count mismatch");
908
909        let mut by_id: HashMap<Uuid, &models::Step> = HashMap::new();
910        for step in &steps_a {
911            by_id.insert(step.step_id, step);
912        }
913
914        for step_b in &steps_b {
915            let Some(step_a) = by_id.get(&step_b.step_id) else {
916                anyhow::bail!("step {} missing in original run", step_b.step_id);
917            };
918            if let (Some(tokens_a), Some(tokens_b)) = (step_a.actual_tokens, step_b.actual_tokens) {
919                if tokens_a > 0 {
920                    let diff = ((tokens_a - tokens_b).abs() as f64) / (tokens_a as f64);
921                    assert!(diff < 0.01, "token drift {:?}", diff);
922                }
923            }
924            if let (Some(cost_a), Some(cost_b)) = (step_a.actual_cost, step_b.actual_cost) {
925                if cost_a > 0.0 {
926                    let diff = ((cost_a - cost_b).abs()) / cost_a;
927                    assert!(diff < 0.01, "cost drift {:?}", diff);
928                }
929            }
930        }
931
932        scheduler_handle.abort();
933        api_handle.abort();
934        drop(container);
935        std::env::remove_var("FLEETFORGE_API_TOKENS");
936        std::env::remove_var("FLEETFORGE_API_ADDR");
937        std::env::remove_var("FLEETFORGE_HTTP_ADDR");
938
939        Ok(())
940    }
941
942    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
943    async fn golden_path_langgraph_capability_and_attestations() -> AnyResult<()> {
944        if !docker_available() {
945            eprintln!("Skipping LangGraph trust test because Docker is unavailable");
946            return Ok(());
947        }
948
949        run_agent_adapter_smoke(AgentAdapterTestConfig {
950            adapter_kind: "langgraph",
951            entrypoint: "tests.python.simple_langgraph:create_graph",
952            slug: "langgraph_demo",
953            initial_state: json!({ "topic": "trust mesh" }),
954            adapter_inputs: None,
955            attestation_key: b"langgraph-test-key",
956        })
957        .await
958    }
959
960    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
961    async fn adapter_smoke_autogen_attestations() -> AnyResult<()> {
962        if !docker_available() {
963            eprintln!("Skipping AutoGen smoke test because Docker is unavailable");
964            return Ok(());
965        }
966
967        run_agent_adapter_smoke(AgentAdapterTestConfig {
968            adapter_kind: "autogen",
969            entrypoint: "tests.python.autogen_stub:create_adapter",
970            slug: "autogen_demo",
971            initial_state: Value::Null,
972            adapter_inputs: Some(json!({ "task": "document adapter contracts" })),
973            attestation_key: b"autogen-smoke-key",
974        })
975        .await
976    }
977
978    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
979    async fn adapter_smoke_crewai_attestations() -> AnyResult<()> {
980        if !docker_available() {
981            eprintln!("Skipping CrewAI smoke test because Docker is unavailable");
982            return Ok(());
983        }
984
985        run_agent_adapter_smoke(AgentAdapterTestConfig {
986            adapter_kind: "crewai",
987            entrypoint: "tests.python.crewai_stub:create_adapter",
988            slug: "crewai_demo",
989            initial_state: Value::Null,
990            adapter_inputs: Some(json!({ "inputs": { "topic": "trust mesh" } })),
991            attestation_key: b"crewai-smoke-key",
992        })
993        .await
994    }
995
996    struct EnvVarGuard {
997        key: String,
998        previous: Option<String>,
999    }
1000
1001    impl EnvVarGuard {
1002        fn set(key: &str, value: &str) -> Self {
1003            let previous = std::env::var(key).ok();
1004            std::env::set_var(key, value);
1005            Self {
1006                key: key.to_string(),
1007                previous,
1008            }
1009        }
1010    }
1011
1012    impl Drop for EnvVarGuard {
1013        fn drop(&mut self) {
1014            if let Some(value) = &self.previous {
1015                std::env::set_var(&self.key, value);
1016            } else {
1017                std::env::remove_var(&self.key);
1018            }
1019        }
1020    }
1021
1022    #[derive(Clone)]
1023    struct AgentAdapterTestConfig {
1024        adapter_kind: &'static str,
1025        entrypoint: &'static str,
1026        slug: &'static str,
1027        initial_state: Value,
1028        adapter_inputs: Option<Value>,
1029        attestation_key: &'static [u8],
1030    }
1031
1032    async fn run_agent_adapter_smoke(config: AgentAdapterTestConfig) -> AnyResult<()> {
1033        let telemetry = TestTelemetry::new();
1034        let docker = Cli::default();
1035        let postgres = docker.run(Postgres::default());
1036        let pg_port = postgres.get_host_port_ipv4(5432);
1037        let database_url = format!("postgres://postgres:postgres@127.0.0.1:{pg_port}/postgres");
1038        let storage = Arc::new(
1039            Storage::connect(StorageConfig {
1040                database_url: database_url.clone(),
1041                max_connections: 5,
1042                connect_timeout: Duration::from_secs(5),
1043                object_store: ObjectStoreConfig::InMemory,
1044            })
1045            .await?,
1046        );
1047        let audit = Arc::new(Audit::new(storage.pool().clone()));
1048        let attestation_vault: Arc<dyn AttestationVault> = Arc::new(
1049            ObjectStoreAttestationVault::new(storage.object_store(), storage.pool().clone()),
1050        );
1051
1052        let mut registry = ExecutorRegistry::new();
1053        registry.register(Arc::new(LangGraphAgentExecutor::new()))?;
1054        let scheduler = Arc::new(Scheduler::new(
1055            Arc::clone(&storage),
1056            Arc::new(registry),
1057            AllowAllPolicy::shared(),
1058            Arc::clone(&audit),
1059            Arc::clone(&attestation_vault),
1060            SchedulerConfig::default(),
1061        ));
1062        let scheduler_handle = tokio::spawn({
1063            let scheduler = Arc::clone(&scheduler);
1064            async move {
1065                scheduler
1066                    .run()
1067                    .await
1068                    .expect("scheduler loop should complete cleanly");
1069            }
1070        });
1071
1072        let grpc_port = reserve_port();
1073        let http_port = reserve_port();
1074        let _api_tokens = EnvVarGuard::set(
1075            "FLEETFORGE_API_TOKENS",
1076            "writer=demo-writer,reader=demo-reader",
1077        );
1078        let _api_addr = EnvVarGuard::set("FLEETFORGE_API_ADDR", &format!("127.0.0.1:{grpc_port}"));
1079        let _http_addr =
1080            EnvVarGuard::set("FLEETFORGE_HTTP_ADDR", &format!("127.0.0.1:{http_port}"));
1081        let attestation_b64 = BASE64.encode(config.attestation_key);
1082        let _trust_guard = EnvVarGuard::set("TRUST_MESH_ALPHA", "1");
1083        let _attestation_guard = EnvVarGuard::set("FLEETFORGE_ATTESTATION_KEY", &attestation_b64);
1084
1085        let api_storage = Arc::clone(&storage);
1086        let api_audit = Arc::clone(&audit);
1087        let version_meta = test_runtime_metadata();
1088        let api_handle = tokio::spawn(async move {
1089            if let Err(err) = start_api_server(api_storage, api_audit, version_meta).await {
1090                eprintln!("API server exited: {err:?}");
1091            }
1092        });
1093
1094        sleep(Duration::from_millis(500)).await;
1095
1096        let endpoint = format!("http://127.0.0.1:{grpc_port}");
1097        let mut client = RuntimeServiceClient::connect(endpoint.clone()).await?;
1098
1099        let repo = repo_root();
1100        let python_paths = vec![
1101            repo.join("sdk/python").to_string_lossy().into_owned(),
1102            repo.join("tests/python").to_string_lossy().into_owned(),
1103        ];
1104        let python_config = json!({ "path": python_paths });
1105        let mut inputs_map = serde_json::Map::new();
1106        inputs_map.insert(
1107            "entrypoint".into(),
1108            Value::String(config.entrypoint.to_string()),
1109        );
1110        inputs_map.insert(
1111            "adapter".into(),
1112            Value::String(config.adapter_kind.to_string()),
1113        );
1114        inputs_map.insert("initial_state".into(), config.initial_state.clone());
1115        inputs_map.insert("python".into(), python_config);
1116        if let Some(extra) = config.adapter_inputs.clone() {
1117            inputs_map.insert("adapter_inputs".into(), extra);
1118        }
1119
1120        let step_id = Uuid::new_v4();
1121        let dag_value = json!({
1122            "steps": [{
1123                "id": step_id,
1124                "type": "agent",
1125                "slug": config.slug,
1126                "inputs": Value::Object(inputs_map.clone()),
1127                "policy": {
1128                    "budget": { "tokens": 128, "cost": 0.005 },
1129                    "priority": 10
1130                }
1131            }],
1132            "edges": []
1133        });
1134        let dag_json = serde_json::to_string(&dag_value)?;
1135        let run_spec = RunSpec {
1136            dag_json,
1137            inputs: None,
1138            seed: 42,
1139            labels: HashMap::from([("suite".into(), "adapter_smoke".into())]),
1140        };
1141
1142        let (run_id, events, _) = submit_and_wait(&mut client, &endpoint, run_spec).await?;
1143        assert!(
1144            events
1145                .iter()
1146                .any(|event| event.tags.iter().any(|tag| tag == "kind:run_succeeded")),
1147            "adapter run should succeed"
1148        );
1149
1150        let run_uuid = Uuid::parse_str(&run_id)?;
1151        let steps = storage.fetch_steps_for_run(run_uuid).await?;
1152        let step = steps
1153            .iter()
1154            .find(|candidate| candidate.step_id == step_id)
1155            .context("agent step not recorded")?;
1156        let snapshot = step
1157            .output_snapshot
1158            .as_ref()
1159            .context("output snapshot missing")?;
1160
1161        let capability_value = snapshot
1162            .get("capability_token")
1163            .context("capability token missing")?;
1164        let token: CapabilityToken = serde_json::from_value(capability_value.clone())?;
1165        verify_capability_token(&token)?;
1166        assert_eq!(token.claims.scope.tool.name, config.slug);
1167
1168        let expected_schema_hash = step
1169            .spec_json
1170            .get("inputs")
1171            .map(digest_json)
1172            .context("spec inputs missing")?;
1173        assert_eq!(token.claims.scope.schema.hash, expected_schema_hash);
1174
1175        let snapshot_token_id = snapshot
1176            .get("capability_token_id")
1177            .and_then(Value::as_str)
1178            .context("capability_token_id missing")?;
1179        assert_eq!(snapshot_token_id, token.token_id().to_string());
1180
1181        let mut tampered = token.clone();
1182        tampered.claims.scope.schema.hash = "bad".to_string();
1183        assert!(
1184            verify_capability_token(&tampered).is_err(),
1185            "tampered capability token should fail verification"
1186        );
1187
1188        let attestation_value = snapshot
1189            .get("attestation")
1190            .context("adapter attestation missing")?;
1191        let attestation_id = verify_adapter_attestation(attestation_value, config.attestation_key)?;
1192        let adapter_ids = snapshot
1193            .get("attestation_ids")
1194            .and_then(Value::as_array)
1195            .context("adapter attestation ids missing")?;
1196        assert!(
1197            adapter_ids
1198                .iter()
1199                .any(|value| value.as_str() == Some(&attestation_id)),
1200            "adapter attestation id should be embedded in outputs"
1201        );
1202
1203        telemetry.force_flush();
1204        let otel_attestation =
1205            telemetry.find_attribute("fleetforge.agent.langgraph", "trust.attestation_id");
1206        assert_eq!(
1207            otel_attestation.as_deref(),
1208            Some(attestation_id.as_str()),
1209            "OTEL span should carry trust.attestation_id"
1210        );
1211
1212        let subject_id = format!("run:{}:step:{}", run_uuid, step_id);
1213        let row = sqlx::query(
1214            "select attestation_id, object_path from attestation_records \
1215             where subject_id = $1 and subject_kind = 'step' order by created_at desc limit 1",
1216        )
1217        .bind(&subject_id)
1218        .fetch_one(storage.pool())
1219        .await?;
1220        let recorded_path: String = row.try_get("object_path")?;
1221        let recorded_attestation = storage
1222            .object_store()
1223            .get(&Path::from(recorded_path.clone()))
1224            .await?
1225            .bytes()
1226            .await?;
1227        let vault_attestation: Attestation = serde_json::from_slice(&recorded_attestation)?;
1228        match vault_attestation.subject {
1229            Some(TrustSubject::Step {
1230                run_id,
1231                step_id: vault_step,
1232            }) => {
1233                assert_eq!(run_id, run_uuid);
1234                assert_eq!(vault_step, step_id);
1235            }
1236            other => anyhow::bail!("unexpected attestation subject {:?}", other),
1237        }
1238
1239        let replay = ReplayExecutor::new(Arc::clone(&storage));
1240        let replay_outcome = replay
1241            .replay_run(
1242                RunId(run_uuid),
1243                ReplayStrategy::Mocked,
1244                DriftTolerance {
1245                    tokens: 0.0,
1246                    cost: 0.0,
1247                },
1248            )
1249            .await?;
1250        assert!(
1251            replay_outcome.all_within_tolerance,
1252            "replay should remain within tolerance"
1253        );
1254
1255        scheduler_handle.abort();
1256        api_handle.abort();
1257        drop(postgres);
1258
1259        telemetry.shutdown();
1260
1261        Ok(())
1262    }
1263
1264    struct TestTelemetry {
1265        spans: Arc<Mutex<Vec<SpanData>>>,
1266        provider: trace::TracerProvider,
1267        _subscriber: DefaultGuard,
1268    }
1269
1270    impl TestTelemetry {
1271        fn new() -> Self {
1272            let spans = Arc::new(Mutex::new(Vec::new()));
1273            let exporter = RecordingExporter::new(Arc::clone(&spans));
1274            let provider = trace::TracerProvider::builder()
1275                .with_simple_exporter(exporter)
1276                .build();
1277            let tracer = provider.tracer("fleetforge-runtime-test", None);
1278            let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
1279            let subscriber = Registry::default()
1280                .with(EnvFilter::new("info"))
1281                .with(otel_layer);
1282            let guard = tracing::subscriber::set_default(subscriber);
1283            Self {
1284                spans,
1285                provider,
1286                _subscriber: guard,
1287            }
1288        }
1289
1290        fn force_flush(&self) {
1291            let _ = self.provider.force_flush();
1292        }
1293
1294        fn find_attribute(&self, span_name: &str, key: &str) -> Option<String> {
1295            let spans = self.spans.lock().expect("span mutex poisoned");
1296            for span in spans.iter() {
1297                if span.name.as_ref() == span_name {
1298                    for (attr_key, value) in span.attributes.iter() {
1299                        if attr_key.as_str() == key {
1300                            return Some(match value {
1301                                OtelValue::String(s) => s.clone().into_owned(),
1302                                OtelValue::Bool(b) => b.to_string(),
1303                                OtelValue::I64(v) => v.to_string(),
1304                                OtelValue::F64(v) => v.to_string(),
1305                                OtelValue::Array(arr) => format!("{arr:?}"),
1306                            });
1307                        }
1308                    }
1309                }
1310            }
1311            None
1312        }
1313
1314        fn shutdown(self) {
1315            drop(self);
1316        }
1317    }
1318
1319    impl Drop for TestTelemetry {
1320        fn drop(&mut self) {
1321            let _ = self.provider.shutdown();
1322            telemetry_otel::shutdown_tracer_provider();
1323        }
1324    }
1325
1326    struct RecordingExporter {
1327        spans: Arc<Mutex<Vec<SpanData>>>,
1328    }
1329
1330    impl RecordingExporter {
1331        fn new(spans: Arc<Mutex<Vec<SpanData>>>) -> Self {
1332            Self { spans }
1333        }
1334    }
1335
1336    #[async_trait::async_trait]
1337    impl SpanExporter for RecordingExporter {
1338        async fn export(&mut self, batch: Vec<SpanData>) -> ExportResult {
1339            let mut guard = self.spans.lock().expect("span mutex poisoned");
1340            guard.extend(batch);
1341            ExportResult::Success
1342        }
1343
1344        fn shutdown(&mut self) -> ExportResult {
1345            self.spans.lock().expect("span mutex poisoned").clear();
1346            ExportResult::Success
1347        }
1348    }
1349
1350    fn canonicalise(value: &Value) -> Value {
1351        match value {
1352            Value::Object(map) => {
1353                let mut ordered = BTreeMap::new();
1354                for (key, value) in map.iter() {
1355                    ordered.insert(key.clone(), canonicalise(value));
1356                }
1357                Value::Object(ordered)
1358            }
1359            Value::Array(items) => Value::Array(items.iter().map(canonicalise).collect()),
1360            other => other.clone(),
1361        }
1362    }
1363
1364    fn canonical_json_bytes(value: &Value) -> Vec<u8> {
1365        let canonical = canonicalise(value);
1366        serde_json::to_vec(&canonical).expect("canonical JSON serialization must succeed")
1367    }
1368
1369    fn verify_adapter_attestation(value: &Value, key: &[u8]) -> AnyResult<String> {
1370        let signature_value = value
1371            .get("signature")
1372            .and_then(Value::as_object)
1373            .and_then(|sig| sig.get("value"))
1374            .and_then(Value::as_str)
1375            .context("attestation signature missing")?;
1376        let statement = value
1377            .get("statement")
1378            .context("attestation statement missing")?;
1379        let mut mac = Hmac::<Sha256>::new_from_slice(key)?;
1380        mac.update(&canonical_json_bytes(statement));
1381        let expected = BASE64.encode(mac.finalize().into_bytes());
1382        anyhow::ensure!(
1383            signature_value == expected,
1384            "adapter attestation signature mismatch"
1385        );
1386        let attestation_id = value
1387            .get("id")
1388            .and_then(Value::as_str)
1389            .context("attestation id missing")?;
1390        Ok(attestation_id.to_string())
1391    }
1392}
1393
1394#[derive(Clone, Copy)]
1395struct RetentionWorkerConfig {
1396    run_days: i64,
1397    artifact_days: i64,
1398    interval: tokio::time::Duration,
1399}
1400
1401fn retention_config_from_env() -> Option<RetentionWorkerConfig> {
1402    let raw_run_env = std::env::var("FLEETFORGE_RETENTION_DAYS").ok()?;
1403    if !feature_allowed(LicensedFeature::LongRetention) {
1404        warn!(
1405            "FLEETFORGE_RETENTION_DAYS is set but the current license does not include extended retention; ignoring."
1406        );
1407        return None;
1408    }
1409    let raw_run = raw_run_env;
1410    let run_days: i64 = raw_run.parse().ok()?;
1411
1412    let artifact_days = std::env::var("FLEETFORGE_RETENTION_ARTIFACT_DAYS")
1413        .ok()
1414        .and_then(|v| v.parse::<i64>().ok())
1415        .unwrap_or(run_days);
1416
1417    if run_days <= 0 && artifact_days <= 0 {
1418        return None;
1419    }
1420
1421    let interval_secs = std::env::var("FLEETFORGE_RETENTION_INTERVAL_SECS")
1422        .ok()
1423        .and_then(|value| value.parse::<u64>().ok())
1424        .unwrap_or(3600)
1425        .max(60);
1426
1427    Some(RetentionWorkerConfig {
1428        run_days,
1429        artifact_days,
1430        interval: tokio::time::Duration::from_secs(interval_secs),
1431    })
1432}
1433
1434fn spawn_retention_worker(storage: Arc<Storage>) {
1435    if let Some(config) = retention_config_from_env() {
1436        let run_window = if config.run_days > 0 {
1437            Some(ChronoDuration::days(config.run_days as i64))
1438        } else {
1439            None
1440        };
1441        info!(
1442            retention_days = config.run_days,
1443            artifact_days = config.artifact_days,
1444            interval_secs = config.interval.as_secs(),
1445            "starting retention scrub worker"
1446        );
1447        let worker_storage = Arc::clone(&storage);
1448        let worker_config = config;
1449        tokio::spawn(async move {
1450            let run_window = run_window;
1451            loop {
1452                if let Some(window) = run_window {
1453                    let cutoff = Utc::now() - window;
1454                    match worker_storage.apply_retention_policy(cutoff).await {
1455                        Ok(stats) => {
1456                            if stats.runs_scrubbed > 0
1457                                || stats.steps_scrubbed > 0
1458                                || stats.events_scrubbed > 0
1459                            {
1460                                info!(
1461                                    runs_scrubbed = stats.runs_scrubbed,
1462                                    steps_scrubbed = stats.steps_scrubbed,
1463                                    events_scrubbed = stats.events_scrubbed,
1464                                    "retention scrub completed"
1465                                );
1466                            }
1467                        }
1468                        Err(err) => {
1469                            warn!(error = %err, "retention scrub failed");
1470                        }
1471                    }
1472                }
1473
1474                let object_store = worker_storage.object_store();
1475                if let Err(err) = purge_retention(
1476                    worker_storage.pool(),
1477                    object_store.as_ref(),
1478                    &StorageRetentionConfig {
1479                        run_days: worker_config.run_days.max(0),
1480                        artifact_days: worker_config.artifact_days.max(0),
1481                    },
1482                )
1483                .await
1484                {
1485                    warn!(error = %err, "retention purge failed");
1486                }
1487
1488                tokio::time::sleep(worker_config.interval).await;
1489            }
1490        });
1491    } else {
1492        info!("retention scrub worker disabled");
1493    }
1494}
1495
1496/// Bootstraps the FleetForge runtime components.
1497pub async fn bootstrap() -> Result<()> {
1498    let telemetry = TelemetryGuard::init("fleetforge-runtime")?;
1499    let storage = Arc::new(Storage::connect_default().await?);
1500    let audit = Arc::new(Audit::new(storage.pool().clone()));
1501    let attestation_vault: Arc<dyn AttestationVault> = Arc::new(ObjectStoreAttestationVault::new(
1502        storage.object_store(),
1503        storage.pool().clone(),
1504    ));
1505    let prompt_registry: Option<Arc<dyn PromptRegistry>> =
1506        match PromptCompiler::from_directory("prompts/agent_packs") {
1507            Ok(registry) => {
1508                info!("loaded prompt packs from ./prompts/agent_packs");
1509                Some(Arc::new(registry) as Arc<dyn PromptRegistry>)
1510            }
1511            Err(err) => {
1512                info!(
1513                    error = %err,
1514                    "prompt registry unavailable; continuing without prompt packs"
1515                );
1516                None
1517            }
1518        };
1519    let mut executor_registry = ExecutorRegistry::new();
1520    let gateways = Arc::new(GatewayRegistry::with_defaults());
1521    executor_registry.register(Arc::new(LlmExecutor::new(
1522        Arc::clone(&gateways),
1523        prompt_registry.clone(),
1524        Option::<Arc<dyn MemoryAdapter>>::None,
1525    )))?;
1526    executor_registry.register(Arc::new(DeterministicReplayExecutor::default()))?;
1527    executor_registry.register(Arc::new(HttpProxyExecutor::new()?))?;
1528    executor_registry.register(Arc::new(LangGraphAgentExecutor::new()))?;
1529
1530    let docker_executor = Arc::new(DockerToolExecutor::new().map_err(|err| {
1531        anyhow!(
1532            "failed to initialise Docker tool executor: {err}. Install Docker or set FLEETFORGE_DOCKER_CLI/FLEETFORGE_TOOLBOX_IMAGE."
1533        )
1534    })?);
1535
1536    let firecracker_executor = match FirecrackerToolExecutor::new() {
1537        Ok(exec) => {
1538            info!("firecracker sandbox enabled for tool execution");
1539            Some(Arc::new(exec))
1540        }
1541        Err(err) => {
1542            info!(error = %err, "firecracker sandbox disabled; continuing with docker-only tools");
1543            None
1544        }
1545    };
1546
1547    executor_registry.register(Arc::new(ToolRouterExecutor::new(
1548        Arc::clone(&docker_executor),
1549        firecracker_executor,
1550    )))?;
1551    #[cfg(feature = "insecure-shell")]
1552    {
1553        match executor::ShellExecutor::new() {
1554            Ok(shell) => executor_registry.register(Arc::new(shell))?,
1555            Err(err) => {
1556                warn!(error = %err, "skipping insecure shell executor registration; enable it explicitly for local demos");
1557            }
1558        }
1559    }
1560    let executors = Arc::new(executor_registry);
1561    let policy_pack: Arc<RuntimePolicyPack> = match std::env::var("FLEETFORGE_POLICY_PACK") {
1562        Ok(value) => match value.trim().to_ascii_lowercase().as_str() {
1563            "default" | "" => {
1564                info!("loading default Trust Mesh policy pack");
1565                Arc::new(RuntimePolicyPack::new(
1566                    "fleetforge.policy.default",
1567                    shared_default_pack()?,
1568                ))
1569            }
1570            "hipaa" => {
1571                info!("loading HIPAA regulated policy pack");
1572                let engine: Arc<dyn PolicyEngine> =
1573                    Arc::new(RegulatedPack::new(RegulatedVertical::Hipaa)?);
1574                Arc::new(RuntimePolicyPack::new(
1575                    "fleetforge.policy.regulated.hipaa",
1576                    engine,
1577                ))
1578            }
1579            "gdpr" => {
1580                info!("loading GDPR regulated policy pack");
1581                let engine: Arc<dyn PolicyEngine> =
1582                    Arc::new(RegulatedPack::new(RegulatedVertical::Gdpr)?);
1583                Arc::new(RuntimePolicyPack::new(
1584                    "fleetforge.policy.regulated.gdpr",
1585                    engine,
1586                ))
1587            }
1588            "allow_all" => {
1589                info!("policy pack set to allow_all (no guardrails enforced)");
1590                Arc::new(RuntimePolicyPack::new(
1591                    "fleetforge.policy.allow_all",
1592                    AllowAllPolicy::shared(),
1593                ))
1594            }
1595            other => return Err(anyhow!("unknown policy pack '{other}'")),
1596        },
1597        Err(_) => {
1598            info!("policy pack not specified; loading default Trust Mesh pack");
1599            Arc::new(RuntimePolicyPack::new(
1600                "fleetforge.policy.default",
1601                shared_default_pack()?,
1602            ))
1603        }
1604    };
1605    let trust_mesh_enabled = trust_mesh_alpha_enabled();
1606    let policy_pack_digest = digest_json(&json!({
1607        "policy_pack": policy_pack.id(),
1608    }));
1609    let metadata = Arc::new(RuntimeMetadata::new(
1610        detect_commit_sha(),
1611        policy_pack.id().to_string(),
1612        policy_pack_digest,
1613        trust_mesh_enabled,
1614    ));
1615
1616    let scheduler = Arc::new(Scheduler::new(
1617        storage.clone(),
1618        executors.clone(),
1619        policy_pack,
1620        Arc::clone(&audit),
1621        Arc::clone(&attestation_vault),
1622        SchedulerConfig::default(),
1623    ));
1624
1625    spawn_retention_worker(storage.clone());
1626
1627    let bus_storage = storage.clone();
1628    let bus_config = bus_config_from_env();
1629    let exporters = match fleetforge_telemetry::exporters::from_env() {
1630        Ok(value) => value,
1631        Err(err) => {
1632            warn!(error = ?err, "failed to initialise external exporters; continuing without them");
1633            Vec::new()
1634        }
1635    };
1636    let bus_handle = tokio::spawn(async move {
1637        run_outbox_forwarder_with_exporters(bus_config, bus_storage, exporters).await
1638    });
1639    let api_storage = storage.clone();
1640    let api_audit = audit.clone();
1641    let api_meta = Arc::clone(&metadata);
1642    let api_handle =
1643        tokio::spawn(async move { start_api_server(api_storage, api_audit, api_meta).await });
1644    let scheduler_handle = tokio::spawn({
1645        let scheduler = scheduler.clone();
1646        async move { scheduler.run().await }
1647    });
1648
1649    let (bus_res, api_res, scheduler_res) = tokio::join!(bus_handle, api_handle, scheduler_handle);
1650
1651    bus_res.map_err(|err| anyhow!("bus task panicked: {err}"))??;
1652    api_res.map_err(|err| anyhow!("api task panicked: {err}"))??;
1653    scheduler_res.map_err(|err| anyhow!("scheduler task panicked: {err}"))??;
1654
1655    // Long-running tasks should keep the process alive; reaching here implies graceful shutdown.
1656    drop(storage);
1657    drop(telemetry);
1658    Ok(())
1659}
1660
1661fn detect_commit_sha() -> String {
1662    const ENV_KEYS: [&str; 5] = [
1663        "FLEETFORGE_COMMIT_SHA",
1664        "VERCEL_GIT_COMMIT_SHA",
1665        "GITHUB_SHA",
1666        "GIT_COMMIT",
1667        "COMMIT_SHA",
1668    ];
1669    for key in ENV_KEYS {
1670        if let Ok(value) = env::var(key) {
1671            let trimmed = value.trim();
1672            if !trimmed.is_empty() {
1673                return trimmed.to_string();
1674            }
1675        }
1676    }
1677    "unknown".to_string()
1678}
1679
1680fn bus_config_from_env() -> BusConfig {
1681    let mut config = BusConfig::default();
1682    if let Ok(bootstrap) = std::env::var("FLEETFORGE_BUS_BOOTSTRAP") {
1683        config.bootstrap_servers = bootstrap;
1684    }
1685    if let Ok(topic) = std::env::var("FLEETFORGE_BUS_TOPIC") {
1686        config.topic = topic;
1687    }
1688
1689    let transactional_id_env = std::env::var("FLEETFORGE_BUS_TRANSACTIONAL_ID")
1690        .ok()
1691        .filter(|id| !id.trim().is_empty());
1692    if let Some(tx_id) = transactional_id_env.clone() {
1693        config.transactional_id = Some(tx_id.clone());
1694    }
1695
1696    if transactional_id_env.is_some() {
1697        info!(
1698            transactional_id = transactional_id_env.as_deref().unwrap_or_default(),
1699            "outbox forwarder running in exactly-once mode; configure consumers with isolation.level=read_committed"
1700        );
1701    } else {
1702        info!(
1703            "outbox forwarder running in at-least-once mode; set FLEETFORGE_BUS_TRANSACTIONAL_ID for exactly-once"
1704        );
1705    }
1706
1707    config
1708}