1pub mod adapters;
4pub mod aibom;
5mod capability;
6pub mod executor;
7pub mod features;
8pub mod gateway;
9pub mod guardrails;
10pub mod memory;
11pub mod model;
12pub mod otel;
13pub mod policy;
14pub mod replay;
15pub mod scheduler;
16
17use std::{env, sync::Arc};
18
19use anyhow::{anyhow, Result};
20use chrono::{Duration as ChronoDuration, Utc};
21use fleetforge_bus::{run_outbox_forwarder_with_exporters, BusConfig};
22use fleetforge_common::licensing::{feature_allowed, LicensedFeature};
23use fleetforge_runtime_api::{start_api_server, RuntimeMetadata};
24use fleetforge_storage::retention::{
25 delete_expired as purge_retention, RetentionConfig as StorageRetentionConfig,
26};
27use fleetforge_storage::Storage;
28use fleetforge_telemetry::{audit::Audit, TelemetryGuard};
29use serde_json::json;
30use tracing::{info, warn};
31
32pub use adapters::{AgentRunAction, AgentRunAdapter, AgentRunContext, AgentRunEnvelope};
33#[cfg(feature = "insecure-shell")]
34pub use executor::ShellExecutor;
35pub use executor::{
36 BudgetCtx, DeterministicReplayExecutor, DockerToolExecutor, ExecutorRegistry,
37 FirecrackerToolExecutor, HttpProxyExecutor, LangGraphAgentExecutor, LlmExecutor, ResourceUsage,
38 StepCtx, StepExecutionResult, StepExecutor, ToolRouterExecutor,
39};
40pub use features::{set_trust_mesh_alpha, trust_mesh_alpha_enabled};
41pub use fleetforge_common::schemas::{
42 AGENT_RUN_SCHEMA_JSON, ARTIFACT_SCHEMA_JSON, RUN_EVENT_SCHEMA_JSON, RUN_SCHEMA_JSON,
43 RUN_SPEC_SCHEMA_JSON, STEP_SCHEMA_JSON,
44};
45pub use fleetforge_common::validation::{
46 validate_agent_run, validate_artifact, validate_run, validate_run_event, validate_run_spec,
47 validate_step_spec, SchemaErrorDetail, SchemaValidationError, StepSpecError,
48 StepSpecErrorDetail,
49};
50pub use fleetforge_policy::{
51 shared_default_pack, AllowAllPolicy, BasicPiiPolicy, Decision as PolicyDecision, PolicyEngine,
52 PolicyRequest, RegulatedPack, RegulatedVertical,
53};
54pub use fleetforge_prompt::{
55 CompiledPrompt, ModelResponse, ModelUsage, PromptCompiler, PromptRegistry,
56};
57pub use fleetforge_trust::{
58 digest_bytes, digest_json, Attestation, AttestationVault, InMemoryAttestationVault,
59 ObjectStoreAttestationVault, Trust, TrustBoundary, TrustDecision, TrustOrigin, TrustSource,
60 TrustSubject, TrustVerdict, Trusted, Untrusted, TRUST_MESH_ALPHA_FLAG,
61};
62pub use gateway::speech::{NoopStt, NoopTts, SpeechToText, TextToSpeech};
63pub use gateway::{
64 anthropic::{AnthropicConfig, AnthropicLanguageModel},
65 openai::{OpenAiConfig, OpenAiLanguageModel},
66 GatewayRegistry, LanguageModel, ToolAdapter,
67};
68pub use guardrails::{OutputGuardPolicy, PolicyBundle, PromptInjectionFallbackPolicy};
69pub use memory::{InMemoryAdapter, MemoryAdapter, MemoryRecord, VectorMatch, VectorMemoryAdapter};
70pub use model::{
71 ChatMessage, ChatRole, ContextSource, LlmInputs, QueuedStep, ResponseSchema, RunId, RunStatus,
72 StepId, StepSpec, StepStatus, StepType, ToolChoice, ToolSpec,
73};
74pub use policy::{enforce_policy, PolicyOutcome, RuntimePolicyPack};
75pub use replay::{
76 DriftTolerance, ReplayExecutor, ReplayOutcome, ReplayStrategy, StepDrift, StepReplay,
77};
78pub use scheduler::{Scheduler, SchedulerConfig};
79
80#[cfg(test)]
81mod memory_test;
82
83#[cfg(test)]
84mod e2e {
85 use super::*;
86 use crate::replay::{DriftTolerance, ReplayExecutor, ReplayStrategy};
87 use anyhow::{anyhow, Context, Result as AnyResult};
88 use base64::engine::general_purpose::STANDARD as BASE64;
89 use base64::Engine;
90 use chrono::Utc;
91 use clickhouse::Client;
92 use fleetforge_common::prost_json::{json_to_prost_struct, prost_struct_to_json};
93 use fleetforge_contracts::runtime::{
94 runtime_service_client::RuntimeServiceClient, GetRunRequest, RunSpec, SubmitRunRequest,
95 };
96 use fleetforge_contracts::tap::{
97 self, subscribe_run_request, tap_service_client::TapServiceClient, RunEventEnvelope,
98 SubscribeRunAck, SubscribeRunOpen, SubscribeRunRequest,
99 };
100 use fleetforge_policy::AllowAllPolicy;
101 use fleetforge_runtime_api::RuntimeMetadata;
102 use fleetforge_storage::{models, object_store::ObjectStoreConfig, StorageConfig};
103 use fleetforge_telemetry::audit::Audit;
104 use fleetforge_telemetry::otel::{self as telemetry_otel, Value as OtelValue};
105 use fleetforge_trust::{verify_capability_token, CapabilityToken, TrustSubject};
106 use hmac::{Hmac, Mac};
107 use object_store::path::Path;
108 use opentelemetry_sdk::export::trace::{ExportResult, SpanData, SpanExporter};
109 use opentelemetry_sdk::trace;
110 use reqwest::StatusCode;
111 use serde_json::{json, Value};
112 use sha2::Sha256;
113 use sqlx::Row;
114 use std::{
115 collections::{BTreeMap, HashMap, HashSet},
116 fs,
117 net::TcpListener,
118 path::PathBuf,
119 sync::{Arc, Mutex},
120 time::Duration,
121 };
122 use testcontainers::clients::Cli;
123 use testcontainers::core::WaitFor;
124 use testcontainers::images::generic::GenericImage;
125 use testcontainers::images::postgres::Postgres;
126 use tokio::sync::mpsc;
127 use tokio::time::sleep;
128 use tokio_stream::wrappers::ReceiverStream;
129 use tonic::metadata::MetadataValue;
130 use tracing::subscriber::DefaultGuard;
131 use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry};
132 use uuid::Uuid;
133
134 struct EchoToolExecutor;
135
136 #[async_trait::async_trait]
137 impl StepExecutor for EchoToolExecutor {
138 fn kind(&self) -> &'static str {
139 "tool"
140 }
141
142 async fn execute(&self, ctx: &StepCtx) -> anyhow::Result<StepExecutionResult> {
143 let command = ctx
144 .untrusted_inputs
145 .as_ref()
146 .get("command")
147 .and_then(Value::as_array)
148 .ok_or_else(|| anyhow!("missing command array"))?;
149 let message = command
150 .iter()
151 .skip(1)
152 .filter_map(Value::as_str)
153 .collect::<Vec<_>>()
154 .join(" ");
155 let stdout = format!("{}\n", message);
156 let output = json!({
157 "stdout": stdout,
158 "stderr": "",
159 "exit_code": 0,
160 });
161
162 Ok(StepExecutionResult::with_usage(
163 output,
164 ResourceUsage {
165 tokens: Some(100),
166 cost: Some(0.001),
167 },
168 ))
169 }
170 }
171
172 fn test_runtime_metadata() -> Arc<RuntimeMetadata> {
173 Arc::new(RuntimeMetadata::new(
174 "test-sha",
175 "fleetforge.policy.default",
176 "test-digest",
177 true,
178 ))
179 }
180
181 fn repo_root() -> PathBuf {
182 PathBuf::from(env!("CARGO_MANIFEST_DIR"))
183 .join("..")
184 .join("..")
185 }
186
187 fn load_agent_team_spec() -> AnyResult<(
188 String,
189 Option<prost_types::Struct>,
190 i64,
191 HashMap<String, String>,
192 )> {
193 let pack_dir = repo_root().join("examples/_packs/demo-pack/agent_team");
194 let spec_value: Value =
195 serde_json::from_str(&fs::read_to_string(pack_dir.join("run_spec.json"))?)?;
196
197 let dag_value = match spec_value.get("dag_json").cloned().unwrap_or(Value::Null) {
198 Value::String(path) if path.starts_with("@file:") => {
199 let dag_path = pack_dir.join(path.trim_start_matches("@file:"));
200 serde_json::from_str(&fs::read_to_string(dag_path)?)?
201 }
202 other => other,
203 };
204
205 let dag_json = serde_json::to_string(&dag_value)?;
206
207 let inputs_struct = match spec_value.get("inputs") {
208 Some(Value::Null) | None => None,
209 Some(value) => Some(json_to_prost_struct(value)?),
210 };
211
212 let seed = spec_value
213 .get("seed")
214 .and_then(Value::as_i64)
215 .ok_or_else(|| anyhow!("run_spec.json must include an explicit integer seed"))?;
216 if seed <= 0 {
217 return Err(anyhow!("run_spec.json seed must be a positive integer"));
218 }
219
220 let labels = spec_value
221 .get("labels")
222 .and_then(Value::as_object)
223 .map(|map| {
224 map.iter()
225 .filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
226 .collect::<HashMap<_, _>>()
227 })
228 .unwrap_or_default();
229
230 Ok((dag_json, inputs_struct, seed, labels))
231 }
232
233 fn reserve_port() -> u16 {
234 TcpListener::bind("127.0.0.1:0")
235 .expect("bind local port")
236 .local_addr()
237 .unwrap()
238 .port()
239 }
240
241 fn docker_available() -> bool {
242 std::env::var("DOCKER_HOST").is_ok() || fs::metadata("/var/run/docker.sock").is_ok()
243 }
244
245 async fn wait_for_clickhouse(client: &Client) -> AnyResult<()> {
246 const RETRIES: usize = 10;
247 for attempt in 0..RETRIES {
248 match client.query("SELECT 1").fetch_one::<(u64,)>().await {
249 Ok(_) => return Ok(()),
250 Err(err) if attempt + 1 < RETRIES => {
251 tokio::time::sleep(Duration::from_millis(200)).await;
252 eprintln!("waiting for ClickHouse readiness: {err}");
253 }
254 Err(err) => return Err(err.into()),
255 }
256 }
257 Ok(())
258 }
259
260 async fn wait_for_run(
261 client: &mut RuntimeServiceClient<tonic::transport::Channel>,
262 run_id: &str,
263 ) -> AnyResult<runtime::RunDetail> {
264 for _ in 0..40 {
265 let mut request = tonic::Request::new(GetRunRequest {
266 run_id: run_id.to_string(),
267 });
268 request.metadata_mut().insert(
269 "authorization",
270 MetadataValue::try_from("Bearer demo-reader")?,
271 );
272 let response = client.get_run(request).await?.into_inner();
273 if let Some(run) = response.run {
274 if run.status == runtime::RunStatus::RunStatusSucceeded as i32 {
275 return Ok(run);
276 }
277 }
278 sleep(Duration::from_millis(250)).await;
279 }
280 Err(anyhow!("run did not succeed in time"))
281 }
282
283 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
284 async fn golden_path_api_surface_acceptance() -> AnyResult<()> {
285 if !docker_available() {
286 eprintln!("Skipping API surface acceptance test because Docker is unavailable");
287 return Ok(());
288 }
289
290 let docker = Cli::default();
291 let postgres = docker.run(Postgres::default());
292 let pg_port = postgres.get_host_port_ipv4(5432);
293 let database_url = format!("postgres://postgres:postgres@127.0.0.1:{pg_port}/postgres");
294 sleep(Duration::from_millis(250)).await;
295
296 let clickhouse_image = GenericImage::new("clickhouse/clickhouse-server", "23.11")
297 .with_wait_for(WaitFor::message("Ready for connections"));
298 let clickhouse = docker.run(clickhouse_image);
299 let ch_port = clickhouse.get_host_port_ipv4(8123);
300 let ch_url = format!("http://127.0.0.1:{ch_port}");
301
302 let admin_client = Client::default()
303 .with_url(ch_url.clone())
304 .with_database("default");
305 wait_for_clickhouse(&admin_client).await?;
306 admin_client
307 .command("CREATE DATABASE IF NOT EXISTS telemetry")
308 .execute()
309 .await?;
310 admin_client
311 .command(
312 "CREATE TABLE IF NOT EXISTS telemetry.run_events (\
313 run_id UUID, \
314 created_at DateTime64(3, 'UTC'), \
315 status String, \
316 attributes JSON\
317 ) ENGINE=MergeTree ORDER BY (created_at, run_id)",
318 )
319 .execute()
320 .await?;
321 admin_client
322 .command(
323 "CREATE TABLE IF NOT EXISTS telemetry.step_events (\
324 run_id UUID, \
325 step_id UUID, \
326 status String, \
327 attempt Int32, \
328 started_at DateTime64(3, 'UTC'), \
329 finished_at Nullable(DateTime64(3, 'UTC')), \
330 attributes JSON\
331 ) ENGINE=MergeTree ORDER BY (started_at, run_id, step_id)",
332 )
333 .execute()
334 .await?;
335
336 let storage = Arc::new(
337 Storage::connect(StorageConfig {
338 database_url,
339 max_connections: 4,
340 connect_timeout: Duration::from_secs(10),
341 object_store: ObjectStoreConfig::InMemory,
342 })
343 .await?,
344 );
345 let audit = Arc::new(Audit::new(storage.pool().clone()));
346 let attestation_vault: Arc<dyn AttestationVault> = Arc::new(
347 ObjectStoreAttestationVault::new(storage.object_store(), storage.pool().clone()),
348 );
349
350 let mut registry = ExecutorRegistry::new();
351 registry.register(Arc::new(EchoToolExecutor))?;
352
353 let scheduler = Arc::new(Scheduler::new(
354 Arc::clone(&storage),
355 Arc::new(registry),
356 AllowAllPolicy::shared(),
357 Arc::clone(&audit),
358 Arc::clone(&attestation_vault),
359 SchedulerConfig::default(),
360 ));
361
362 struct TaskAbortOnDrop(tokio::task::JoinHandle<()>);
363 impl TaskAbortOnDrop {
364 fn abort(&mut self) {
365 self.0.abort();
366 }
367 }
368 impl Drop for TaskAbortOnDrop {
369 fn drop(&mut self) {
370 self.0.abort();
371 }
372 }
373
374 let scheduler_handle = tokio::spawn({
375 let scheduler = Arc::clone(&scheduler);
376 async move {
377 scheduler.run().await.expect("scheduler loop");
378 }
379 });
380 let mut scheduler_handle = TaskAbortOnDrop(scheduler_handle);
381
382 let grpc_port = reserve_port();
383 let http_port = reserve_port();
384 std::env::set_var(
385 "FLEETFORGE_API_TOKENS",
386 "writer=demo-writer,reader=demo-reader",
387 );
388 std::env::set_var("FLEETFORGE_API_ADDR", format!("127.0.0.1:{grpc_port}"));
389 std::env::set_var("FLEETFORGE_HTTP_ADDR", format!("127.0.0.1:{http_port}"));
390 std::env::set_var("CLICKHOUSE_DSN", ch_url.clone());
391 std::env::set_var("CLICKHOUSE_DATABASE", "telemetry");
392 std::env::set_var("CLICKHOUSE_RUN_TABLE", "run_events");
393 std::env::set_var("CLICKHOUSE_STEP_TABLE", "step_events");
394
395 struct EnvCleanup;
396 impl Drop for EnvCleanup {
397 fn drop(&mut self) {
398 for key in [
399 "FLEETFORGE_API_TOKENS",
400 "FLEETFORGE_API_ADDR",
401 "FLEETFORGE_HTTP_ADDR",
402 "CLICKHOUSE_DSN",
403 "CLICKHOUSE_DATABASE",
404 "CLICKHOUSE_RUN_TABLE",
405 "CLICKHOUSE_STEP_TABLE",
406 ] {
407 std::env::remove_var(key);
408 }
409 }
410 }
411 let _env_guard = EnvCleanup;
412
413 let api_storage = Arc::clone(&storage);
414 let api_audit = Arc::clone(&audit);
415 let version_meta = test_runtime_metadata();
416 let api_handle = tokio::spawn(async move {
417 if let Err(err) = start_api_server(api_storage, api_audit, version_meta).await {
418 eprintln!("API server exited: {err:?}");
419 }
420 });
421 let mut api_handle = TaskAbortOnDrop(api_handle);
422
423 sleep(Duration::from_millis(500)).await;
424
425 let dag_step_id = "11111111-1111-4111-8111-111111111111";
426 let dag_value = json!({
427 "steps": [{
428 "id": dag_step_id,
429 "type": "tool",
430 "inputs": {
431 "command": ["echo", "hello", "acceptance"]
432 }
433 }],
434 "edges": []
435 });
436 let dag_json_string = serde_json::to_string(&dag_value)?;
437
438 let http_client = reqwest::Client::new();
439 let submit_body = json!({
440 "dag_json": dag_value,
441 "inputs": {},
442 "seed": 42,
443 "labels": { "purpose": "acceptance" }
444 });
445
446 let submit_response = http_client
447 .post(format!("http://127.0.0.1:{http_port}/v1/runs"))
448 .header("Authorization", "Bearer demo-writer")
449 .json(&submit_body)
450 .send()
451 .await?;
452 assert_eq!(submit_response.status(), StatusCode::CREATED);
453 let submit_json: Value = submit_response.json().await?;
454 let run_id_http = submit_json["run_id"]
455 .as_str()
456 .ok_or_else(|| anyhow!("missing run_id in HTTP response"))?
457 .to_string();
458
459 let detail_response = http_client
460 .get(format!(
461 "http://127.0.0.1:{http_port}/v1/runs/{run_id_http}"
462 ))
463 .header("Authorization", "Bearer demo-reader")
464 .send()
465 .await?;
466 assert_eq!(detail_response.status(), StatusCode::OK);
467 let detail_json: Value = detail_response.json().await?;
468 assert_eq!(
469 detail_json["seed"].as_i64(),
470 Some(42),
471 "HTTP readback should preserve seed"
472 );
473
474 let endpoint = format!("http://127.0.0.1:{grpc_port}");
475 let mut runtime_client = RuntimeServiceClient::connect(endpoint.clone()).await?;
476 let run_detail = wait_for_run(&mut runtime_client, &run_id_http).await?;
477 assert_eq!(
478 run_detail.seed, 42,
479 "gRPC GetRun should return the same seed"
480 );
481 let run_uuid = Uuid::parse_str(&run_id_http)?;
482 let steps = storage
483 .fetch_steps_for_run(run_uuid)
484 .await
485 .context("fetch steps for run")?;
486 let first_step = steps.first().context("expected recorded step")?;
487
488 let mut tap_client = TapServiceClient::connect(endpoint.clone()).await?;
489 let (mut tx, rx) = mpsc::channel::<SubscribeRunRequest>(32);
490 tx.send(SubscribeRunRequest {
491 request: Some(subscribe_run_request::Request::Open(SubscribeRunOpen {
492 run_ids: vec![run_id_http.clone()],
493 step_ids: Vec::new(),
494 kinds: Vec::new(),
495 severities: Vec::new(),
496 tags: Vec::new(),
497 since_offset: 0,
498 batch_size: 128,
499 max_inflight: 64,
500 heartbeat_interval_ms: 1_000,
501 cursor: "".into(),
502 })),
503 })
504 .await?;
505 let outbound = ReceiverStream::new(rx);
506 let mut tap_request = tonic::Request::new(outbound);
507 tap_request.metadata_mut().insert(
508 "authorization",
509 MetadataValue::try_from("Bearer demo-reader")?,
510 );
511 let mut stream = tap_client.subscribe_run(tap_request).await?.into_inner();
512 let mut saw_event = false;
513 while let Some(mut response) = stream.message().await? {
514 if response.heartbeat {
515 continue;
516 }
517 saw_event = true;
518 let offset = response.offset;
519 if offset > 0 {
520 tx.send(SubscribeRunRequest {
521 request: Some(subscribe_run_request::Request::Ack(SubscribeRunAck {
522 offset,
523 })),
524 })
525 .await?;
526 }
527 break;
528 }
529 drop(tx);
530 assert!(saw_event, "expected tap stream to yield events");
531
532 #[derive(Clone, serde::Serialize, clickhouse::Row)]
533 struct RunEventInsert {
534 run_id: Uuid,
535 created_at: chrono::DateTime<Utc>,
536 status: String,
537 attributes: Value,
538 }
539
540 #[derive(Clone, serde::Serialize, clickhouse::Row)]
541 struct StepEventInsert {
542 run_id: Uuid,
543 step_id: Uuid,
544 status: String,
545 attempt: i32,
546 started_at: chrono::DateTime<Utc>,
547 finished_at: Option<chrono::DateTime<Utc>>,
548 attributes: Value,
549 }
550
551 let telemetry_client = Client::default()
552 .with_url(ch_url.clone())
553 .with_database("telemetry");
554 let now = Utc::now();
555
556 let mut run_insert = telemetry_client.insert("telemetry.run_events")?;
557 run_insert
558 .write(&RunEventInsert {
559 run_id: run_uuid,
560 created_at: now,
561 status: "succeeded".to_string(),
562 attributes: json!({ "source": "acceptance" }),
563 })
564 .await?;
565 run_insert.end().await?;
566
567 let mut step_insert = telemetry_client.insert("telemetry.step_events")?;
568 step_insert
569 .write(&StepEventInsert {
570 run_id: run_uuid,
571 step_id: first_step.step_id,
572 status: "succeeded".to_string(),
573 attempt: first_step.attempt,
574 started_at: first_step.created_at,
575 finished_at: Some(Utc::now()),
576 attributes: json!({ "source": "acceptance" }),
577 })
578 .await?;
579 step_insert.end().await?;
580
581 let mut tap_client = TapServiceClient::connect(endpoint.clone()).await?;
582 let mut query_request = tap::QueryEventsRequest {
583 run_ids: vec![run_id_http.clone()],
584 workspace_ids: Vec::new(),
585 kinds: Vec::new(),
586 policy_effects: Vec::new(),
587 start_time: None,
588 end_time: None,
589 page_size: 10,
590 page_token: String::new(),
591 source: tap::QueryEventSource::QueryEventSourceRun as i32,
592 step_ids: Vec::new(),
593 ascending: true,
594 severities: Vec::new(),
595 };
596 let mut query_req = tonic::Request::new(query_request);
597 query_req.metadata_mut().insert(
598 "authorization",
599 MetadataValue::try_from("Bearer demo-reader")?,
600 );
601 let query_response = tap_client.query_events(query_req).await?.into_inner();
602 assert!(
603 !query_response.events.is_empty(),
604 "expected ClickHouse-backed QueryEvents response"
605 );
606
607 let mut labels = HashMap::new();
608 labels.insert("purpose".to_string(), "breakpoint".to_string());
609 let run2_spec = RunSpec {
610 dag_json: dag_json_string.clone(),
611 inputs: None,
612 seed: 99,
613 labels,
614 breakpoints: vec![dag_step_id.to_string()],
615 ..Default::default()
616 };
617
618 let mut submit_req = tonic::Request::new(SubmitRunRequest {
619 spec: Some(run2_spec),
620 idempotency_key: String::new(),
621 });
622 submit_req.metadata_mut().insert(
623 "authorization",
624 MetadataValue::try_from("Bearer demo-writer")?,
625 );
626 let run2_id = runtime_client
627 .submit_run(submit_req)
628 .await?
629 .into_inner()
630 .run_id;
631
632 let mut paused = false;
633 for _ in 0..40 {
634 let mut get_req = tonic::Request::new(GetRunRequest {
635 run_id: run2_id.clone(),
636 });
637 get_req.metadata_mut().insert(
638 "authorization",
639 MetadataValue::try_from("Bearer demo-reader")?,
640 );
641 let response = runtime_client.get_run(get_req).await?.into_inner();
642 if let Some(run) = response.run {
643 if run.status == runtime::RunStatus::RunStatusPausedAtStep as i32 {
644 paused = true;
645 break;
646 }
647 }
648 sleep(Duration::from_millis(250)).await;
649 }
650 assert!(paused, "run should pause at declared breakpoint");
651
652 let mut update_req = tonic::Request::new(runtime::UpdateBreakpointsRequest {
653 run_id: run2_id.clone(),
654 add: Vec::new(),
655 remove: Vec::new(),
656 clear_all: true,
657 });
658 update_req.metadata_mut().insert(
659 "authorization",
660 MetadataValue::try_from("Bearer demo-writer")?,
661 );
662 let update_response = runtime_client
663 .update_breakpoints(update_req)
664 .await?
665 .into_inner();
666 assert!(
667 update_response.breakpoints.is_empty(),
668 "clear_all should remove pending breakpoints"
669 );
670
671 let mut resume_req = tonic::Request::new(runtime::ResumeRunRequest {
672 run_id: run2_id.clone(),
673 });
674 resume_req.metadata_mut().insert(
675 "authorization",
676 MetadataValue::try_from("Bearer demo-writer")?,
677 );
678 let resume_response = runtime_client.resume_run(resume_req).await?.into_inner();
679 assert_eq!(
680 resume_response.status,
681 runtime::RunStatus::RunStatusRunning as i32,
682 "resume should transition run back to running"
683 );
684
685 let _ = wait_for_run(&mut runtime_client, &run2_id).await?;
686
687 scheduler_handle.abort();
688 api_handle.abort();
689 drop(clickhouse);
690 drop(postgres);
691
692 Ok(())
693 }
694
695 async fn submit_and_wait(
696 client: &mut RuntimeServiceClient<tonic::transport::Channel>,
697 tap_endpoint: &str,
698 spec: RunSpec,
699 ) -> AnyResult<(String, Vec<RunEventEnvelope>, runtime::RunDetail)> {
700 let submit_request = SubmitRunRequest {
701 spec: Some(spec),
702 idempotency_key: String::new(),
703 };
704
705 let mut request = tonic::Request::new(submit_request);
706 request.metadata_mut().insert(
707 "authorization",
708 MetadataValue::try_from("Bearer demo-writer")?,
709 );
710
711 let response = client.submit_run(request).await?.into_inner();
712 let run_id = response.run_id;
713
714 let mut tap_client = TapServiceClient::connect(tap_endpoint.to_string()).await?;
715 let (mut tx, rx) = mpsc::channel::<SubscribeRunRequest>(32);
716 tx.send(SubscribeRunRequest {
717 request: Some(subscribe_run_request::Request::Open(SubscribeRunOpen {
718 run_ids: vec![run_id.clone()],
719 step_ids: Vec::new(),
720 kinds: Vec::new(),
721 severities: Vec::new(),
722 tags: Vec::new(),
723 since_offset: 0,
724 batch_size: 256,
725 max_inflight: 64,
726 heartbeat_interval_ms: 1_000,
727 })),
728 })
729 .await?;
730
731 let outbound = ReceiverStream::new(rx);
732 let mut tap_request = tonic::Request::new(outbound);
733 tap_request.metadata_mut().insert(
734 "authorization",
735 MetadataValue::try_from("Bearer demo-reader")?,
736 );
737
738 let mut stream = tap_client.subscribe_run(tap_request).await?.into_inner();
739 let mut events = Vec::new();
740
741 while let Some(mut response) = stream.message().await? {
742 if response.heartbeat {
743 continue;
744 }
745
746 if let Some(event) = response.event.take() {
747 if response.offset > 0 {
748 let ack = SubscribeRunRequest {
749 request: Some(subscribe_run_request::Request::Ack(SubscribeRunAck {
750 offset: response.offset,
751 })),
752 };
753 if tx.send(ack).await.is_err() {
754 break;
755 }
756 }
757
758 let done = event.tags.iter().any(|tag| tag == "kind:run_succeeded");
759 events.push(event);
760 if done {
761 break;
762 }
763 }
764 }
765
766 let run_detail = wait_for_run(client, &run_id).await?;
767 Ok((run_id, events, run_detail))
768 }
769
770 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
771 async fn golden_path_agent_team_replay_within_one_percent() -> AnyResult<()> {
772 if std::fs::metadata("/var/run/docker.sock").is_err()
773 && std::env::var("DOCKER_HOST").is_err()
774 {
775 eprintln!("Skipping E2E test because Docker is unavailable");
776 return Ok(());
777 }
778
779 let docker = Cli::default();
780 let container = docker.run(Postgres::default());
781 let port = container.get_host_port_ipv4(5432);
782 let database_url = format!("postgres://postgres:postgres@127.0.0.1:{port}/postgres");
783 sleep(Duration::from_millis(250)).await;
784
785 let storage = Arc::new(
786 Storage::connect(StorageConfig {
787 database_url,
788 max_connections: 4,
789 connect_timeout: Duration::from_secs(10),
790 object_store: ObjectStoreConfig::InMemory,
791 })
792 .await?,
793 );
794 let audit = Arc::new(Audit::new(storage.pool().clone()));
795
796 let mut registry = ExecutorRegistry::new();
797 registry.register(Arc::new(EchoToolExecutor))?;
798
799 let scheduler = Arc::new(Scheduler::new(
800 Arc::clone(&storage),
801 Arc::new(registry),
802 AllowAllPolicy::shared(),
803 Arc::clone(&audit),
804 Arc::clone(&attestation_vault),
805 SchedulerConfig::default(),
806 ));
807
808 let scheduler_handle = tokio::spawn({
809 let scheduler = Arc::clone(&scheduler);
810 async move {
811 scheduler.run().await.expect("scheduler loop");
812 }
813 });
814
815 let grpc_port = reserve_port();
816 let http_port = reserve_port();
817 std::env::set_var(
818 "FLEETFORGE_API_TOKENS",
819 "writer=demo-writer,reader=demo-reader",
820 );
821 std::env::set_var("FLEETFORGE_API_ADDR", format!("127.0.0.1:{grpc_port}"));
822 std::env::set_var("FLEETFORGE_HTTP_ADDR", format!("127.0.0.1:{http_port}"));
823
824 let api_storage = Arc::clone(&storage);
825 let api_audit = Arc::clone(&audit);
826 let version_meta = test_runtime_metadata();
827 let api_handle = tokio::spawn(async move {
828 if let Err(err) = start_api_server(api_storage, api_audit, version_meta).await {
829 eprintln!("API server exited: {err:?}");
830 }
831 });
832
833 sleep(Duration::from_millis(500)).await;
834
835 let endpoint = format!("http://127.0.0.1:{grpc_port}");
836 let channel = tonic::transport::Channel::from_shared(endpoint.clone())?
837 .connect()
838 .await?;
839 let mut client = RuntimeServiceClient::new(channel);
840
841 let (dag_json, inputs_struct, seed, labels) = load_agent_team_spec()?;
842 let run_spec = RunSpec {
843 dag_json: dag_json.clone(),
844 inputs: inputs_struct.clone(),
845 seed,
846 labels: labels.clone(),
847 };
848
849 let (run_a, events_a, _) =
850 submit_and_wait(&mut client, &endpoint, run_spec.clone()).await?;
851 assert!(events_a
852 .iter()
853 .any(|event| event.tags.iter().any(|tag| tag == "kind:run_succeeded")));
854
855 let trace_ids: HashSet<_> = events_a
856 .iter()
857 .filter_map(|event| event.trace.as_ref())
858 .map(|trace| trace.trace_id.clone())
859 .filter(|id| !id.is_empty())
860 .collect();
861 assert_eq!(
862 trace_ids.len(),
863 1,
864 "expected a single trace_id across the agent run"
865 );
866
867 let mut saw_budget = false;
868 let mut saw_policy = false;
869 for event in &events_a {
870 if let Some(raw) = event.raw.as_ref() {
871 let raw_json = prost_struct_to_json(raw);
872 if raw_json
873 .get("budget")
874 .and_then(|value| value.as_object())
875 .is_some()
876 {
877 saw_budget = true;
878 }
879 if raw_json
880 .get("policy_decisions")
881 .and_then(|value| value.as_object())
882 .is_some()
883 {
884 saw_policy = true;
885 }
886 }
887 }
888 assert!(saw_budget, "expected budget telemetry in stream payloads");
889 assert!(
890 saw_policy,
891 "expected policy decision summaries in stream payloads"
892 );
893
894 let steps_a = storage
895 .fetch_steps_for_run(Uuid::parse_str(&run_a)?)
896 .await?;
897
898 let (run_b, events_b, _) = submit_and_wait(&mut client, &endpoint, run_spec).await?;
899 assert!(events_b
900 .iter()
901 .any(|event| event.tags.iter().any(|tag| tag == "kind:run_succeeded")));
902
903 let steps_b = storage
904 .fetch_steps_for_run(Uuid::parse_str(&run_b)?)
905 .await?;
906
907 assert_eq!(steps_a.len(), steps_b.len(), "step count mismatch");
908
909 let mut by_id: HashMap<Uuid, &models::Step> = HashMap::new();
910 for step in &steps_a {
911 by_id.insert(step.step_id, step);
912 }
913
914 for step_b in &steps_b {
915 let Some(step_a) = by_id.get(&step_b.step_id) else {
916 anyhow::bail!("step {} missing in original run", step_b.step_id);
917 };
918 if let (Some(tokens_a), Some(tokens_b)) = (step_a.actual_tokens, step_b.actual_tokens) {
919 if tokens_a > 0 {
920 let diff = ((tokens_a - tokens_b).abs() as f64) / (tokens_a as f64);
921 assert!(diff < 0.01, "token drift {:?}", diff);
922 }
923 }
924 if let (Some(cost_a), Some(cost_b)) = (step_a.actual_cost, step_b.actual_cost) {
925 if cost_a > 0.0 {
926 let diff = ((cost_a - cost_b).abs()) / cost_a;
927 assert!(diff < 0.01, "cost drift {:?}", diff);
928 }
929 }
930 }
931
932 scheduler_handle.abort();
933 api_handle.abort();
934 drop(container);
935 std::env::remove_var("FLEETFORGE_API_TOKENS");
936 std::env::remove_var("FLEETFORGE_API_ADDR");
937 std::env::remove_var("FLEETFORGE_HTTP_ADDR");
938
939 Ok(())
940 }
941
942 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
943 async fn golden_path_langgraph_capability_and_attestations() -> AnyResult<()> {
944 if !docker_available() {
945 eprintln!("Skipping LangGraph trust test because Docker is unavailable");
946 return Ok(());
947 }
948
949 run_agent_adapter_smoke(AgentAdapterTestConfig {
950 adapter_kind: "langgraph",
951 entrypoint: "tests.python.simple_langgraph:create_graph",
952 slug: "langgraph_demo",
953 initial_state: json!({ "topic": "trust mesh" }),
954 adapter_inputs: None,
955 attestation_key: b"langgraph-test-key",
956 })
957 .await
958 }
959
960 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
961 async fn adapter_smoke_autogen_attestations() -> AnyResult<()> {
962 if !docker_available() {
963 eprintln!("Skipping AutoGen smoke test because Docker is unavailable");
964 return Ok(());
965 }
966
967 run_agent_adapter_smoke(AgentAdapterTestConfig {
968 adapter_kind: "autogen",
969 entrypoint: "tests.python.autogen_stub:create_adapter",
970 slug: "autogen_demo",
971 initial_state: Value::Null,
972 adapter_inputs: Some(json!({ "task": "document adapter contracts" })),
973 attestation_key: b"autogen-smoke-key",
974 })
975 .await
976 }
977
978 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
979 async fn adapter_smoke_crewai_attestations() -> AnyResult<()> {
980 if !docker_available() {
981 eprintln!("Skipping CrewAI smoke test because Docker is unavailable");
982 return Ok(());
983 }
984
985 run_agent_adapter_smoke(AgentAdapterTestConfig {
986 adapter_kind: "crewai",
987 entrypoint: "tests.python.crewai_stub:create_adapter",
988 slug: "crewai_demo",
989 initial_state: Value::Null,
990 adapter_inputs: Some(json!({ "inputs": { "topic": "trust mesh" } })),
991 attestation_key: b"crewai-smoke-key",
992 })
993 .await
994 }
995
996 struct EnvVarGuard {
997 key: String,
998 previous: Option<String>,
999 }
1000
1001 impl EnvVarGuard {
1002 fn set(key: &str, value: &str) -> Self {
1003 let previous = std::env::var(key).ok();
1004 std::env::set_var(key, value);
1005 Self {
1006 key: key.to_string(),
1007 previous,
1008 }
1009 }
1010 }
1011
1012 impl Drop for EnvVarGuard {
1013 fn drop(&mut self) {
1014 if let Some(value) = &self.previous {
1015 std::env::set_var(&self.key, value);
1016 } else {
1017 std::env::remove_var(&self.key);
1018 }
1019 }
1020 }
1021
1022 #[derive(Clone)]
1023 struct AgentAdapterTestConfig {
1024 adapter_kind: &'static str,
1025 entrypoint: &'static str,
1026 slug: &'static str,
1027 initial_state: Value,
1028 adapter_inputs: Option<Value>,
1029 attestation_key: &'static [u8],
1030 }
1031
1032 async fn run_agent_adapter_smoke(config: AgentAdapterTestConfig) -> AnyResult<()> {
1033 let telemetry = TestTelemetry::new();
1034 let docker = Cli::default();
1035 let postgres = docker.run(Postgres::default());
1036 let pg_port = postgres.get_host_port_ipv4(5432);
1037 let database_url = format!("postgres://postgres:postgres@127.0.0.1:{pg_port}/postgres");
1038 let storage = Arc::new(
1039 Storage::connect(StorageConfig {
1040 database_url: database_url.clone(),
1041 max_connections: 5,
1042 connect_timeout: Duration::from_secs(5),
1043 object_store: ObjectStoreConfig::InMemory,
1044 })
1045 .await?,
1046 );
1047 let audit = Arc::new(Audit::new(storage.pool().clone()));
1048 let attestation_vault: Arc<dyn AttestationVault> = Arc::new(
1049 ObjectStoreAttestationVault::new(storage.object_store(), storage.pool().clone()),
1050 );
1051
1052 let mut registry = ExecutorRegistry::new();
1053 registry.register(Arc::new(LangGraphAgentExecutor::new()))?;
1054 let scheduler = Arc::new(Scheduler::new(
1055 Arc::clone(&storage),
1056 Arc::new(registry),
1057 AllowAllPolicy::shared(),
1058 Arc::clone(&audit),
1059 Arc::clone(&attestation_vault),
1060 SchedulerConfig::default(),
1061 ));
1062 let scheduler_handle = tokio::spawn({
1063 let scheduler = Arc::clone(&scheduler);
1064 async move {
1065 scheduler
1066 .run()
1067 .await
1068 .expect("scheduler loop should complete cleanly");
1069 }
1070 });
1071
1072 let grpc_port = reserve_port();
1073 let http_port = reserve_port();
1074 let _api_tokens = EnvVarGuard::set(
1075 "FLEETFORGE_API_TOKENS",
1076 "writer=demo-writer,reader=demo-reader",
1077 );
1078 let _api_addr = EnvVarGuard::set("FLEETFORGE_API_ADDR", &format!("127.0.0.1:{grpc_port}"));
1079 let _http_addr =
1080 EnvVarGuard::set("FLEETFORGE_HTTP_ADDR", &format!("127.0.0.1:{http_port}"));
1081 let attestation_b64 = BASE64.encode(config.attestation_key);
1082 let _trust_guard = EnvVarGuard::set("TRUST_MESH_ALPHA", "1");
1083 let _attestation_guard = EnvVarGuard::set("FLEETFORGE_ATTESTATION_KEY", &attestation_b64);
1084
1085 let api_storage = Arc::clone(&storage);
1086 let api_audit = Arc::clone(&audit);
1087 let version_meta = test_runtime_metadata();
1088 let api_handle = tokio::spawn(async move {
1089 if let Err(err) = start_api_server(api_storage, api_audit, version_meta).await {
1090 eprintln!("API server exited: {err:?}");
1091 }
1092 });
1093
1094 sleep(Duration::from_millis(500)).await;
1095
1096 let endpoint = format!("http://127.0.0.1:{grpc_port}");
1097 let mut client = RuntimeServiceClient::connect(endpoint.clone()).await?;
1098
1099 let repo = repo_root();
1100 let python_paths = vec![
1101 repo.join("sdk/python").to_string_lossy().into_owned(),
1102 repo.join("tests/python").to_string_lossy().into_owned(),
1103 ];
1104 let python_config = json!({ "path": python_paths });
1105 let mut inputs_map = serde_json::Map::new();
1106 inputs_map.insert(
1107 "entrypoint".into(),
1108 Value::String(config.entrypoint.to_string()),
1109 );
1110 inputs_map.insert(
1111 "adapter".into(),
1112 Value::String(config.adapter_kind.to_string()),
1113 );
1114 inputs_map.insert("initial_state".into(), config.initial_state.clone());
1115 inputs_map.insert("python".into(), python_config);
1116 if let Some(extra) = config.adapter_inputs.clone() {
1117 inputs_map.insert("adapter_inputs".into(), extra);
1118 }
1119
1120 let step_id = Uuid::new_v4();
1121 let dag_value = json!({
1122 "steps": [{
1123 "id": step_id,
1124 "type": "agent",
1125 "slug": config.slug,
1126 "inputs": Value::Object(inputs_map.clone()),
1127 "policy": {
1128 "budget": { "tokens": 128, "cost": 0.005 },
1129 "priority": 10
1130 }
1131 }],
1132 "edges": []
1133 });
1134 let dag_json = serde_json::to_string(&dag_value)?;
1135 let run_spec = RunSpec {
1136 dag_json,
1137 inputs: None,
1138 seed: 42,
1139 labels: HashMap::from([("suite".into(), "adapter_smoke".into())]),
1140 };
1141
1142 let (run_id, events, _) = submit_and_wait(&mut client, &endpoint, run_spec).await?;
1143 assert!(
1144 events
1145 .iter()
1146 .any(|event| event.tags.iter().any(|tag| tag == "kind:run_succeeded")),
1147 "adapter run should succeed"
1148 );
1149
1150 let run_uuid = Uuid::parse_str(&run_id)?;
1151 let steps = storage.fetch_steps_for_run(run_uuid).await?;
1152 let step = steps
1153 .iter()
1154 .find(|candidate| candidate.step_id == step_id)
1155 .context("agent step not recorded")?;
1156 let snapshot = step
1157 .output_snapshot
1158 .as_ref()
1159 .context("output snapshot missing")?;
1160
1161 let capability_value = snapshot
1162 .get("capability_token")
1163 .context("capability token missing")?;
1164 let token: CapabilityToken = serde_json::from_value(capability_value.clone())?;
1165 verify_capability_token(&token)?;
1166 assert_eq!(token.claims.scope.tool.name, config.slug);
1167
1168 let expected_schema_hash = step
1169 .spec_json
1170 .get("inputs")
1171 .map(digest_json)
1172 .context("spec inputs missing")?;
1173 assert_eq!(token.claims.scope.schema.hash, expected_schema_hash);
1174
1175 let snapshot_token_id = snapshot
1176 .get("capability_token_id")
1177 .and_then(Value::as_str)
1178 .context("capability_token_id missing")?;
1179 assert_eq!(snapshot_token_id, token.token_id().to_string());
1180
1181 let mut tampered = token.clone();
1182 tampered.claims.scope.schema.hash = "bad".to_string();
1183 assert!(
1184 verify_capability_token(&tampered).is_err(),
1185 "tampered capability token should fail verification"
1186 );
1187
1188 let attestation_value = snapshot
1189 .get("attestation")
1190 .context("adapter attestation missing")?;
1191 let attestation_id = verify_adapter_attestation(attestation_value, config.attestation_key)?;
1192 let adapter_ids = snapshot
1193 .get("attestation_ids")
1194 .and_then(Value::as_array)
1195 .context("adapter attestation ids missing")?;
1196 assert!(
1197 adapter_ids
1198 .iter()
1199 .any(|value| value.as_str() == Some(&attestation_id)),
1200 "adapter attestation id should be embedded in outputs"
1201 );
1202
1203 telemetry.force_flush();
1204 let otel_attestation =
1205 telemetry.find_attribute("fleetforge.agent.langgraph", "trust.attestation_id");
1206 assert_eq!(
1207 otel_attestation.as_deref(),
1208 Some(attestation_id.as_str()),
1209 "OTEL span should carry trust.attestation_id"
1210 );
1211
1212 let subject_id = format!("run:{}:step:{}", run_uuid, step_id);
1213 let row = sqlx::query(
1214 "select attestation_id, object_path from attestation_records \
1215 where subject_id = $1 and subject_kind = 'step' order by created_at desc limit 1",
1216 )
1217 .bind(&subject_id)
1218 .fetch_one(storage.pool())
1219 .await?;
1220 let recorded_path: String = row.try_get("object_path")?;
1221 let recorded_attestation = storage
1222 .object_store()
1223 .get(&Path::from(recorded_path.clone()))
1224 .await?
1225 .bytes()
1226 .await?;
1227 let vault_attestation: Attestation = serde_json::from_slice(&recorded_attestation)?;
1228 match vault_attestation.subject {
1229 Some(TrustSubject::Step {
1230 run_id,
1231 step_id: vault_step,
1232 }) => {
1233 assert_eq!(run_id, run_uuid);
1234 assert_eq!(vault_step, step_id);
1235 }
1236 other => anyhow::bail!("unexpected attestation subject {:?}", other),
1237 }
1238
1239 let replay = ReplayExecutor::new(Arc::clone(&storage));
1240 let replay_outcome = replay
1241 .replay_run(
1242 RunId(run_uuid),
1243 ReplayStrategy::Mocked,
1244 DriftTolerance {
1245 tokens: 0.0,
1246 cost: 0.0,
1247 },
1248 )
1249 .await?;
1250 assert!(
1251 replay_outcome.all_within_tolerance,
1252 "replay should remain within tolerance"
1253 );
1254
1255 scheduler_handle.abort();
1256 api_handle.abort();
1257 drop(postgres);
1258
1259 telemetry.shutdown();
1260
1261 Ok(())
1262 }
1263
1264 struct TestTelemetry {
1265 spans: Arc<Mutex<Vec<SpanData>>>,
1266 provider: trace::TracerProvider,
1267 _subscriber: DefaultGuard,
1268 }
1269
1270 impl TestTelemetry {
1271 fn new() -> Self {
1272 let spans = Arc::new(Mutex::new(Vec::new()));
1273 let exporter = RecordingExporter::new(Arc::clone(&spans));
1274 let provider = trace::TracerProvider::builder()
1275 .with_simple_exporter(exporter)
1276 .build();
1277 let tracer = provider.tracer("fleetforge-runtime-test", None);
1278 let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
1279 let subscriber = Registry::default()
1280 .with(EnvFilter::new("info"))
1281 .with(otel_layer);
1282 let guard = tracing::subscriber::set_default(subscriber);
1283 Self {
1284 spans,
1285 provider,
1286 _subscriber: guard,
1287 }
1288 }
1289
1290 fn force_flush(&self) {
1291 let _ = self.provider.force_flush();
1292 }
1293
1294 fn find_attribute(&self, span_name: &str, key: &str) -> Option<String> {
1295 let spans = self.spans.lock().expect("span mutex poisoned");
1296 for span in spans.iter() {
1297 if span.name.as_ref() == span_name {
1298 for (attr_key, value) in span.attributes.iter() {
1299 if attr_key.as_str() == key {
1300 return Some(match value {
1301 OtelValue::String(s) => s.clone().into_owned(),
1302 OtelValue::Bool(b) => b.to_string(),
1303 OtelValue::I64(v) => v.to_string(),
1304 OtelValue::F64(v) => v.to_string(),
1305 OtelValue::Array(arr) => format!("{arr:?}"),
1306 });
1307 }
1308 }
1309 }
1310 }
1311 None
1312 }
1313
1314 fn shutdown(self) {
1315 drop(self);
1316 }
1317 }
1318
1319 impl Drop for TestTelemetry {
1320 fn drop(&mut self) {
1321 let _ = self.provider.shutdown();
1322 telemetry_otel::shutdown_tracer_provider();
1323 }
1324 }
1325
1326 struct RecordingExporter {
1327 spans: Arc<Mutex<Vec<SpanData>>>,
1328 }
1329
1330 impl RecordingExporter {
1331 fn new(spans: Arc<Mutex<Vec<SpanData>>>) -> Self {
1332 Self { spans }
1333 }
1334 }
1335
1336 #[async_trait::async_trait]
1337 impl SpanExporter for RecordingExporter {
1338 async fn export(&mut self, batch: Vec<SpanData>) -> ExportResult {
1339 let mut guard = self.spans.lock().expect("span mutex poisoned");
1340 guard.extend(batch);
1341 ExportResult::Success
1342 }
1343
1344 fn shutdown(&mut self) -> ExportResult {
1345 self.spans.lock().expect("span mutex poisoned").clear();
1346 ExportResult::Success
1347 }
1348 }
1349
1350 fn canonicalise(value: &Value) -> Value {
1351 match value {
1352 Value::Object(map) => {
1353 let mut ordered = BTreeMap::new();
1354 for (key, value) in map.iter() {
1355 ordered.insert(key.clone(), canonicalise(value));
1356 }
1357 Value::Object(ordered)
1358 }
1359 Value::Array(items) => Value::Array(items.iter().map(canonicalise).collect()),
1360 other => other.clone(),
1361 }
1362 }
1363
1364 fn canonical_json_bytes(value: &Value) -> Vec<u8> {
1365 let canonical = canonicalise(value);
1366 serde_json::to_vec(&canonical).expect("canonical JSON serialization must succeed")
1367 }
1368
1369 fn verify_adapter_attestation(value: &Value, key: &[u8]) -> AnyResult<String> {
1370 let signature_value = value
1371 .get("signature")
1372 .and_then(Value::as_object)
1373 .and_then(|sig| sig.get("value"))
1374 .and_then(Value::as_str)
1375 .context("attestation signature missing")?;
1376 let statement = value
1377 .get("statement")
1378 .context("attestation statement missing")?;
1379 let mut mac = Hmac::<Sha256>::new_from_slice(key)?;
1380 mac.update(&canonical_json_bytes(statement));
1381 let expected = BASE64.encode(mac.finalize().into_bytes());
1382 anyhow::ensure!(
1383 signature_value == expected,
1384 "adapter attestation signature mismatch"
1385 );
1386 let attestation_id = value
1387 .get("id")
1388 .and_then(Value::as_str)
1389 .context("attestation id missing")?;
1390 Ok(attestation_id.to_string())
1391 }
1392}
1393
1394#[derive(Clone, Copy)]
1395struct RetentionWorkerConfig {
1396 run_days: i64,
1397 artifact_days: i64,
1398 interval: tokio::time::Duration,
1399}
1400
1401fn retention_config_from_env() -> Option<RetentionWorkerConfig> {
1402 let raw_run_env = std::env::var("FLEETFORGE_RETENTION_DAYS").ok()?;
1403 if !feature_allowed(LicensedFeature::LongRetention) {
1404 warn!(
1405 "FLEETFORGE_RETENTION_DAYS is set but the current license does not include extended retention; ignoring."
1406 );
1407 return None;
1408 }
1409 let raw_run = raw_run_env;
1410 let run_days: i64 = raw_run.parse().ok()?;
1411
1412 let artifact_days = std::env::var("FLEETFORGE_RETENTION_ARTIFACT_DAYS")
1413 .ok()
1414 .and_then(|v| v.parse::<i64>().ok())
1415 .unwrap_or(run_days);
1416
1417 if run_days <= 0 && artifact_days <= 0 {
1418 return None;
1419 }
1420
1421 let interval_secs = std::env::var("FLEETFORGE_RETENTION_INTERVAL_SECS")
1422 .ok()
1423 .and_then(|value| value.parse::<u64>().ok())
1424 .unwrap_or(3600)
1425 .max(60);
1426
1427 Some(RetentionWorkerConfig {
1428 run_days,
1429 artifact_days,
1430 interval: tokio::time::Duration::from_secs(interval_secs),
1431 })
1432}
1433
1434fn spawn_retention_worker(storage: Arc<Storage>) {
1435 if let Some(config) = retention_config_from_env() {
1436 let run_window = if config.run_days > 0 {
1437 Some(ChronoDuration::days(config.run_days as i64))
1438 } else {
1439 None
1440 };
1441 info!(
1442 retention_days = config.run_days,
1443 artifact_days = config.artifact_days,
1444 interval_secs = config.interval.as_secs(),
1445 "starting retention scrub worker"
1446 );
1447 let worker_storage = Arc::clone(&storage);
1448 let worker_config = config;
1449 tokio::spawn(async move {
1450 let run_window = run_window;
1451 loop {
1452 if let Some(window) = run_window {
1453 let cutoff = Utc::now() - window;
1454 match worker_storage.apply_retention_policy(cutoff).await {
1455 Ok(stats) => {
1456 if stats.runs_scrubbed > 0
1457 || stats.steps_scrubbed > 0
1458 || stats.events_scrubbed > 0
1459 {
1460 info!(
1461 runs_scrubbed = stats.runs_scrubbed,
1462 steps_scrubbed = stats.steps_scrubbed,
1463 events_scrubbed = stats.events_scrubbed,
1464 "retention scrub completed"
1465 );
1466 }
1467 }
1468 Err(err) => {
1469 warn!(error = %err, "retention scrub failed");
1470 }
1471 }
1472 }
1473
1474 let object_store = worker_storage.object_store();
1475 if let Err(err) = purge_retention(
1476 worker_storage.pool(),
1477 object_store.as_ref(),
1478 &StorageRetentionConfig {
1479 run_days: worker_config.run_days.max(0),
1480 artifact_days: worker_config.artifact_days.max(0),
1481 },
1482 )
1483 .await
1484 {
1485 warn!(error = %err, "retention purge failed");
1486 }
1487
1488 tokio::time::sleep(worker_config.interval).await;
1489 }
1490 });
1491 } else {
1492 info!("retention scrub worker disabled");
1493 }
1494}
1495
1496pub async fn bootstrap() -> Result<()> {
1498 let telemetry = TelemetryGuard::init("fleetforge-runtime")?;
1499 let storage = Arc::new(Storage::connect_default().await?);
1500 let audit = Arc::new(Audit::new(storage.pool().clone()));
1501 let attestation_vault: Arc<dyn AttestationVault> = Arc::new(ObjectStoreAttestationVault::new(
1502 storage.object_store(),
1503 storage.pool().clone(),
1504 ));
1505 let prompt_registry: Option<Arc<dyn PromptRegistry>> =
1506 match PromptCompiler::from_directory("prompts/agent_packs") {
1507 Ok(registry) => {
1508 info!("loaded prompt packs from ./prompts/agent_packs");
1509 Some(Arc::new(registry) as Arc<dyn PromptRegistry>)
1510 }
1511 Err(err) => {
1512 info!(
1513 error = %err,
1514 "prompt registry unavailable; continuing without prompt packs"
1515 );
1516 None
1517 }
1518 };
1519 let mut executor_registry = ExecutorRegistry::new();
1520 let gateways = Arc::new(GatewayRegistry::with_defaults());
1521 executor_registry.register(Arc::new(LlmExecutor::new(
1522 Arc::clone(&gateways),
1523 prompt_registry.clone(),
1524 Option::<Arc<dyn MemoryAdapter>>::None,
1525 )))?;
1526 executor_registry.register(Arc::new(DeterministicReplayExecutor::default()))?;
1527 executor_registry.register(Arc::new(HttpProxyExecutor::new()?))?;
1528 executor_registry.register(Arc::new(LangGraphAgentExecutor::new()))?;
1529
1530 let docker_executor = Arc::new(DockerToolExecutor::new().map_err(|err| {
1531 anyhow!(
1532 "failed to initialise Docker tool executor: {err}. Install Docker or set FLEETFORGE_DOCKER_CLI/FLEETFORGE_TOOLBOX_IMAGE."
1533 )
1534 })?);
1535
1536 let firecracker_executor = match FirecrackerToolExecutor::new() {
1537 Ok(exec) => {
1538 info!("firecracker sandbox enabled for tool execution");
1539 Some(Arc::new(exec))
1540 }
1541 Err(err) => {
1542 info!(error = %err, "firecracker sandbox disabled; continuing with docker-only tools");
1543 None
1544 }
1545 };
1546
1547 executor_registry.register(Arc::new(ToolRouterExecutor::new(
1548 Arc::clone(&docker_executor),
1549 firecracker_executor,
1550 )))?;
1551 #[cfg(feature = "insecure-shell")]
1552 {
1553 match executor::ShellExecutor::new() {
1554 Ok(shell) => executor_registry.register(Arc::new(shell))?,
1555 Err(err) => {
1556 warn!(error = %err, "skipping insecure shell executor registration; enable it explicitly for local demos");
1557 }
1558 }
1559 }
1560 let executors = Arc::new(executor_registry);
1561 let policy_pack: Arc<RuntimePolicyPack> = match std::env::var("FLEETFORGE_POLICY_PACK") {
1562 Ok(value) => match value.trim().to_ascii_lowercase().as_str() {
1563 "default" | "" => {
1564 info!("loading default Trust Mesh policy pack");
1565 Arc::new(RuntimePolicyPack::new(
1566 "fleetforge.policy.default",
1567 shared_default_pack()?,
1568 ))
1569 }
1570 "hipaa" => {
1571 info!("loading HIPAA regulated policy pack");
1572 let engine: Arc<dyn PolicyEngine> =
1573 Arc::new(RegulatedPack::new(RegulatedVertical::Hipaa)?);
1574 Arc::new(RuntimePolicyPack::new(
1575 "fleetforge.policy.regulated.hipaa",
1576 engine,
1577 ))
1578 }
1579 "gdpr" => {
1580 info!("loading GDPR regulated policy pack");
1581 let engine: Arc<dyn PolicyEngine> =
1582 Arc::new(RegulatedPack::new(RegulatedVertical::Gdpr)?);
1583 Arc::new(RuntimePolicyPack::new(
1584 "fleetforge.policy.regulated.gdpr",
1585 engine,
1586 ))
1587 }
1588 "allow_all" => {
1589 info!("policy pack set to allow_all (no guardrails enforced)");
1590 Arc::new(RuntimePolicyPack::new(
1591 "fleetforge.policy.allow_all",
1592 AllowAllPolicy::shared(),
1593 ))
1594 }
1595 other => return Err(anyhow!("unknown policy pack '{other}'")),
1596 },
1597 Err(_) => {
1598 info!("policy pack not specified; loading default Trust Mesh pack");
1599 Arc::new(RuntimePolicyPack::new(
1600 "fleetforge.policy.default",
1601 shared_default_pack()?,
1602 ))
1603 }
1604 };
1605 let trust_mesh_enabled = trust_mesh_alpha_enabled();
1606 let policy_pack_digest = digest_json(&json!({
1607 "policy_pack": policy_pack.id(),
1608 }));
1609 let metadata = Arc::new(RuntimeMetadata::new(
1610 detect_commit_sha(),
1611 policy_pack.id().to_string(),
1612 policy_pack_digest,
1613 trust_mesh_enabled,
1614 ));
1615
1616 let scheduler = Arc::new(Scheduler::new(
1617 storage.clone(),
1618 executors.clone(),
1619 policy_pack,
1620 Arc::clone(&audit),
1621 Arc::clone(&attestation_vault),
1622 SchedulerConfig::default(),
1623 ));
1624
1625 spawn_retention_worker(storage.clone());
1626
1627 let bus_storage = storage.clone();
1628 let bus_config = bus_config_from_env();
1629 let exporters = match fleetforge_telemetry::exporters::from_env() {
1630 Ok(value) => value,
1631 Err(err) => {
1632 warn!(error = ?err, "failed to initialise external exporters; continuing without them");
1633 Vec::new()
1634 }
1635 };
1636 let bus_handle = tokio::spawn(async move {
1637 run_outbox_forwarder_with_exporters(bus_config, bus_storage, exporters).await
1638 });
1639 let api_storage = storage.clone();
1640 let api_audit = audit.clone();
1641 let api_meta = Arc::clone(&metadata);
1642 let api_handle =
1643 tokio::spawn(async move { start_api_server(api_storage, api_audit, api_meta).await });
1644 let scheduler_handle = tokio::spawn({
1645 let scheduler = scheduler.clone();
1646 async move { scheduler.run().await }
1647 });
1648
1649 let (bus_res, api_res, scheduler_res) = tokio::join!(bus_handle, api_handle, scheduler_handle);
1650
1651 bus_res.map_err(|err| anyhow!("bus task panicked: {err}"))??;
1652 api_res.map_err(|err| anyhow!("api task panicked: {err}"))??;
1653 scheduler_res.map_err(|err| anyhow!("scheduler task panicked: {err}"))??;
1654
1655 drop(storage);
1657 drop(telemetry);
1658 Ok(())
1659}
1660
1661fn detect_commit_sha() -> String {
1662 const ENV_KEYS: [&str; 5] = [
1663 "FLEETFORGE_COMMIT_SHA",
1664 "VERCEL_GIT_COMMIT_SHA",
1665 "GITHUB_SHA",
1666 "GIT_COMMIT",
1667 "COMMIT_SHA",
1668 ];
1669 for key in ENV_KEYS {
1670 if let Ok(value) = env::var(key) {
1671 let trimmed = value.trim();
1672 if !trimmed.is_empty() {
1673 return trimmed.to_string();
1674 }
1675 }
1676 }
1677 "unknown".to_string()
1678}
1679
1680fn bus_config_from_env() -> BusConfig {
1681 let mut config = BusConfig::default();
1682 if let Ok(bootstrap) = std::env::var("FLEETFORGE_BUS_BOOTSTRAP") {
1683 config.bootstrap_servers = bootstrap;
1684 }
1685 if let Ok(topic) = std::env::var("FLEETFORGE_BUS_TOPIC") {
1686 config.topic = topic;
1687 }
1688
1689 let transactional_id_env = std::env::var("FLEETFORGE_BUS_TRANSACTIONAL_ID")
1690 .ok()
1691 .filter(|id| !id.trim().is_empty());
1692 if let Some(tx_id) = transactional_id_env.clone() {
1693 config.transactional_id = Some(tx_id.clone());
1694 }
1695
1696 if transactional_id_env.is_some() {
1697 info!(
1698 transactional_id = transactional_id_env.as_deref().unwrap_or_default(),
1699 "outbox forwarder running in exactly-once mode; configure consumers with isolation.level=read_committed"
1700 );
1701 } else {
1702 info!(
1703 "outbox forwarder running in at-least-once mode; set FLEETFORGE_BUS_TRANSACTIONAL_ID for exactly-once"
1704 );
1705 }
1706
1707 config
1708}