{ "$schema": "https://ferrum-os.vercel.app/schemas/benchmarks-v2.schema.json", "schemaVersion": 2, "project": "FerrumOS", "releaseVersion": "0.1.1", "researchVersion": "1.0.0", "source": { "repository": "https://github.com/VyomKulshrestha/Ferrum-OS", "commit": "84926841a401760b869af158ee80b0e709d0d6af", "snapshotDate": "2026-08-14", "sha256": "618ebe473f70e66a68e19a7fdf627ea0ec513b82914b14d38dc4562fe6059bff", "raw": "https://raw.githubusercontent.com/VyomKulshrestha/Ferrum-OS/main/benchmarks.json" }, "benchmarks": [ { "id": "os-safety-fixture-rules-jepa", "metric": "balanced_accuracy", "value": 0.8140000000000001, "unit": "ratio", "sampleSize": 500, "falseNegativeRate": 0.20799999999999996, "falsePositiveRate": 0.16400000000000003, "protocol": "authored-balanced-counterfactual-fixture", "boundary": "Authored balanced safety fixture; no material JEPA safety advantage over the per-action mean baseline was established." }, { "id": "os-safety-fixture-rules-action-mean", "metric": "balanced_accuracy", "value": 0.812, "unit": "ratio", "sampleSize": 500, "protocol": "authored-balanced-counterfactual-fixture", "boundary": "Simple baseline; no material JEPA advantage established." }, { "id": "ring3-preview-h1", "metric": "mean_preview_latency", "range": [ 1290, 1400 ], "unit": "microseconds", "samplesPerRun": 100, "runs": 3, "protocol": "qemu-whpx-ring3-preview", "boundary": "Excludes provider inference, action dispatch, execution, and approval latency." }, { "id": "ring3-preview-h5", "metric": "mean_preview_latency", "range": [ 1430, 1570 ], "unit": "microseconds", "samplesPerRun": 100, "runs": 3, "protocol": "qemu-whpx-ring3-preview", "boundary": "Bounded lookahead in the same QEMU/WHPX profile." }, { "id": "physical-jepa-simulator", "metric": "balanced_accuracy", "value": 0.9944006975286777, "unit": "ratio", "sampleSize": 2500, "falseNegatives": 1, "falsePositives": 16, "protocol": "deterministic-physical-simulator", "boundary": "Deterministic simulator evidence; the artifact is permanently shadow-only and has no actuator authority." }, { "id": "neural-synthetic-contract", "metric": "contract_trials", "signalTrials": 600, "acceptedSignalAccuracy": 1, "artifactTrials": 400, "artifactAbstentionRate": 1, "noControlWindows": 10000, "emittedIntents": 0, "protocol": "deterministic-synthetic-eeg-contract", "boundary": [ "Synthetic decoder evidence only; no human EEG accuracy or usability claim.", "No-control emitted intents are decoder candidates; OS commit additionally requires pairing, calibration, a local non-neural arm, signed preview, and revision checks.", "Physical neural intents remain proposal-only and cannot invoke an adapter." ] }, { "id": "qemu-command-audit", "metric": "passing_command_paths", "focusedCases": 101, "focusedPassed": 101, "catalogEntries": 81, "catalogPassed": 81, "unknownCommandPaths": 0, "protocol": "ferrumos-qemu-command-audit-v1", "boundary": "This is a dated emulator command-path audit for the named source commit and artifact hashes. It is not evidence of broad physical-PC compatibility, hardware accuracy, live EEG performance, formal safety, or independent replication." }, { "id": "cyber-physical-software-regression", "metric": "passing_software_contracts", "contractTestsPassed": 152, "contractTestsFailed": 0, "modelAndDecoderGatesPassed": 32, "modelAndDecoderGatesFailed": 0, "sourceCommit": "167b04766ee5ea3c5714cd09a551feb7bc78f359", "protocol": "cyber-physical-software-regression-2026-08-14", "coveredSoftwareBoundaries": [ "versioned provenance and logical clocks", "deterministic sessions replay and fault injection", "virtual sensor actuator EEG watchdog and stop contracts", "scripted Gazebo ROS 2 and Webots bridge boundaries", "deterministic supervisor watchdog recovery and stop priority", "ROS 2 MQTT and CAN transport conformance", "actuator-disabled delivery", "bounded neural intent and proposal-only physical goals", "shadow-only physical JEPA evaluation", "host-managed agent-cell contract", "privacy retention reliability and incident primitives" ], "boundary": "Local deterministic software regression on the named commit. It does not prove an installed Gazebo or Webots deployment, real ROS 2/MQTT/CAN infrastructure, physical clocks, actuator hardware, robot safety, live EEG, hard-real-time behavior, native hypervisor containment, certification, or independent replication." } ], "protocols": { "$schema": "https://raw.githubusercontent.com/VyomKulshrestha/Ferrum-OS/main/schemas/benchmarks.schema.json", "benchmark_scope": { "comparability": "Each section has a distinct protocol; cross-section ranking is invalid.", "latest_tagged_software_release": "v0.1.1", "source_channel": "current main evidence plus frozen research artifacts" }, "canonical_url": "https://github.com/VyomKulshrestha/Ferrum-OS/blob/main/benchmarks.json", "current_ring3_preview": { "accelerator": "whpx", "evidence_grade": "repeated-emulator-measurement", "heap_growth_bytes_each_run": [ 0, 0, 0 ], "horizons": [ { "blocked_previews_each_run": [ 2, 2, 2 ], "horizon": 1, "mean_microseconds_range": [ 1290, 1400 ], "median_microseconds_across_runs": [ 1000, 1000, 1000 ], "median_run_mean_microseconds": 1320, "p95_microseconds_across_runs": [ 2000, 2000, 2000 ] }, { "blocked_previews_each_run": [ 2, 2, 2 ], "horizon": 2, "mean_microseconds_range": [ 1350, 1490 ], "median_microseconds_across_runs": [ 1000, 1000, 1000 ], "median_run_mean_microseconds": 1390, "p95_microseconds_across_runs": [ 2000, 2000, 2000 ] }, { "blocked_previews_each_run": [ 2, 2, 2 ], "horizon": 3, "mean_microseconds_range": [ 1370, 1450 ], "median_microseconds_across_runs": [ 1000, 1000, 1000 ], "median_run_mean_microseconds": 1430, "p95_microseconds_across_runs": [ 2000, 2000, 2000 ] }, { "blocked_previews_each_run": [ 2, 2, 2 ], "horizon": 4, "mean_microseconds_range": [ 1400, 1470 ], "median_microseconds_across_runs": [ 1000, 1000, 1000 ], "median_run_mean_microseconds": 1430, "p95_microseconds_across_runs": [ 2000, 2000, 2000 ] }, { "blocked_previews_each_run": [ 2, 2, 2 ], "horizon": 5, "mean_microseconds_range": [ 1430, 1570 ], "median_microseconds_across_runs": [ 1000, 1000, 2000 ], "median_run_mean_microseconds": 1510, "p95_microseconds_across_runs": [ 2000, 2000, 2000 ] } ], "iterations_per_horizon_per_run": 100, "model_load_milliseconds_range": [ 26, 30 ], "protocol_id": "ring3-preview-whpx-2026-08-13", "runs": 3, "scope": "ring-3 Heliox capture + encoder + transition + safety predicate preview; no action dispatch", "source_commit": "54e1cc1819d3b3c11436f437733d197d469d1607", "warmup_previews_per_run": 64 }, "cyber_physical_software": { "claim_boundary": "Local deterministic software regression on the named commit. It does not prove an installed Gazebo or Webots deployment, real ROS 2/MQTT/CAN infrastructure, physical clocks, actuator hardware, robot safety, live EEG, hard-real-time behavior, native hypervisor containment, certification, or independent replication.", "contract_suites": [ { "command": "cargo test --manifest-path userland/physical-runtime/Cargo.toml --target x86_64-pc-windows-msvc", "component": "ferrum-physical-runtime", "failed": 0, "passed": 117 }, { "command": "cargo test --manifest-path userland/neural-protocol/Cargo.toml --target x86_64-pc-windows-msvc", "component": "ferrum-neural-protocol", "failed": 0, "passed": 9 }, { "command": "python -m unittest discover -s tools/neurod -p test_*.py", "component": "neurod", "failed": 0, "passed": 15 }, { "command": "python -m unittest tools.physical_sim_bridge.test_bridge", "component": "physical-simulator-bridge", "failed": 0, "passed": 11 } ], "contract_tests_failed": 0, "contract_tests_passed": 152, "covered_software_boundaries": [ "versioned provenance and logical clocks", "deterministic sessions replay and fault injection", "virtual sensor actuator EEG watchdog and stop contracts", "scripted Gazebo ROS 2 and Webots bridge boundaries", "deterministic supervisor watchdog recovery and stop priority", "ROS 2 MQTT and CAN transport conformance", "actuator-disabled delivery", "bounded neural intent and proposal-only physical goals", "shadow-only physical JEPA evaluation", "host-managed agent-cell contract", "privacy retention reliability and incident primitives" ], "environment": { "host_os": "Windows 11", "python_runtime": "CPython", "rust_target": "x86_64-pc-windows-msvc" }, "evidence_grade": "local-deterministic-regression", "model_and_decoder_gates": [ { "command": "python scripts/verify_physical_world_model.py --quick", "component": "physical-world-model", "failed": 0, "passed": 21 }, { "command": "python scripts/verify_physical_jepa_robustness.py", "component": "physical-jepa-robustness", "failed": 0, "passed": 8 }, { "command": "python scripts/evaluate_neural_simulator.py --output target/neural-simulator-audit.json", "component": "neural-simulator", "failed": 0, "passed": 3 } ], "model_and_decoder_gates_failed": 0, "model_and_decoder_gates_passed": 32, "protocol_id": "cyber-physical-software-regression-2026-08-14", "source_commit": "167b04766ee5ea3c5714cd09a551feb7bc78f359" }, "environment": { "accelerator": "WHPX", "host_os": "Windows 11", "qemu_version": "10.1.0", "queue_guest_ram_mb": 4096, "runtime_guest_ram_mb": 512 }, "global_limitations": [ "The paper, physical simulator, neural synthetic, QEMU, and cyber-physical software evaluations use different protocols and are not directly comparable.", "No live EEG, human neural calibration, robot hardware-in-the-loop, or broad physical-PC benchmark is claimed.", "Ring-3 timing uses a virtualized 1 kHz PIT; provider, tool execution, and operator-confirmation latency are excluded.", "A passing benchmark is evidence for its named fixture and protocol, not formal safety proof." ], "metric_definitions": { "balanced_accuracy": { "definition": "Mean of sensitivity and specificity within the named fixture.", "higher_is_better": true, "range": [ 0, 1 ], "unit": "ratio" }, "false_negative_rate": { "lower_is_better": true, "range": [ 0, 1 ], "unit": "ratio" }, "false_positive_rate": { "lower_is_better": true, "range": [ 0, 1 ], "unit": "ratio" }, "preview_latency": { "clock": "virtualized 1 kHz PIT with raw TSC retained in source runs", "lower_is_better": true, "unit": "microseconds" }, "queue_batch_latency": { "lower_is_better": true, "request_count": 96, "unit": "milliseconds" } }, "neural_synthetic": { "accepted_signal_accuracy": 1, "artifact_abstention_rate": 1, "artifact_trials": 400, "claim_boundary": [ "Synthetic decoder evidence only; no human EEG accuracy or usability claim.", "No-control emitted intents are decoder candidates; OS commit additionally requires pairing, calibration, a local non-neural arm, signed preview, and revision checks.", "Physical neural intents remain proposal-only and cannot invoke an adapter." ], "emitted_intents": 0, "evidence_grade": "deterministic-synthetic-signal", "no_control_windows": 10000, "passed": true, "protocol_id": "neural-ssvep-synthetic-v1", "signal_trials": 600 }, "paired_preview_queue": { "baseline_batch_milliseconds": [ 15114.109, 14910.766, 15086.329 ], "baseline_median_batch_milliseconds": 15086.329, "baseline_source_commit": "54e1cc1819d3b3c11436f437733d197d469d1607", "evidence_grade": "paired-repeated-emulator-measurement", "execution_records_added_each_run": [ 0, 0, 0 ], "guest_fault_free_each_run": [ true, true, true ], "limitation": "The single-threaded daemon serializes preview inference; concurrency here means multiple outstanding requests with response correlation, not parallel neural execution.", "median_improvement_percent": 10.4, "optimized_batch_milliseconds": [ 13784.147, 13496.023, 13517.414 ], "optimized_median_batch_milliseconds": 13517.414, "optimized_median_milliseconds_per_serialized_request": 140.806, "optimized_source_commit": "c92056d8635af5e4ee2a81351350b1b25cfd4861", "protocol_id": "paired-preview-queue-whpx-2026-08-13", "requests_per_run": 96, "responses_received_each_run": [ 96, 96, 96 ], "runs_each": 3 }, "paper_release": { "accepted_rows": 13697, "claim_boundary": "Authored balanced safety fixture; no material JEPA safety advantage over the per-action mean baseline was established.", "eligible_transitions": 13270, "episodes": 3639, "evidence_commit": "42ea7b8", "evidence_grade": "authored-balanced-fixture", "five_pipeline_balanced_accuracy_mean": 0.7976, "fixture_episodes": 500, "protocol_id": "world-model-study-v1.0.0/episode-disjoint-500", "release_tag": "world-model-study-v1.0.0", "rules_plus_jepa_balanced_accuracy": 0.8140000000000001, "rules_plus_jepa_false_negative_rate": 0.20799999999999996, "rules_plus_jepa_false_positive_rate": 0.16400000000000003, "rules_plus_mean_balanced_accuracy": 0.812 }, "physical_simulator_jepa": { "claim_boundary": "Deterministic simulator evidence; the artifact is permanently shadow-only and has no actuator authority.", "episodes": 2500, "evidence_grade": "deterministic-simulator", "false_negatives": 1, "false_positives": 16, "normalized_rollout_error": { "per_action_mean_h1": 0.019730109721422195, "per_action_mean_h2": 0.034468431025743484, "per_action_mean_h3": 0.04665490239858627, "per_action_mean_h4": 0.05753195658326149, "per_action_mean_h5": 0.06636197119951248, "physical_jepa_h1": 0.004541031550616026, "physical_jepa_h2": 0.007690879516303539, "physical_jepa_h3": 0.010492870584130287, "physical_jepa_h4": 0.01306567620486021, "physical_jepa_h5": 0.015461564064025879 }, "protocol_id": "physical-jepa-simulator-v1", "rules_plus_jepa_balanced_accuracy": 0.9944006975286777, "transitions": 15000, "validated_for_gating": false }, "provenance": { "generator": "scripts/generate_public_evidence.py", "inputs": [ { "path": "docs/benchmarks/raw/2026-08-13/manifest.json", "sha256": "030e41206f386e3318e06135fa5f2ce651948984ec39d7e4e212a30d9776b876" }, { "path": "docs/benchmarks/raw/2026-08-13/runtime-run-1.json", "sha256": "699f099e6f87b203b08d84e848ee039e96d71126c21d4c94c7e5a223ea9a1523" }, { "path": "docs/benchmarks/raw/2026-08-13/runtime-run-2.json", "sha256": "c857c0deb89d0afb0723016c2daf75d0607eda982a1e84bb283fcd4fa759aa26" }, { "path": "docs/benchmarks/raw/2026-08-13/runtime-run-3.json", "sha256": "267db39b3ac2a59bd2502c3102680f7e2bfcd50395d919350700bf3e28c8c19b" }, { "path": "docs/benchmarks/raw/2026-08-13/concurrency-baseline-run-1.json", "sha256": "11cde8b42902c57310e58d5da80259eed120ca3654e18cc5e96cca767c1aea01" }, { "path": "docs/benchmarks/raw/2026-08-13/concurrency-baseline-run-2.json", "sha256": "2e507a785a5368f30c8ad823bd02d5229b69452f7ebc26ff421026b41131b369" }, { "path": "docs/benchmarks/raw/2026-08-13/concurrency-baseline-run-3.json", "sha256": "668f162e9872294e4ed2dfd859448d01e7cce3f4a46c307676edd754a9d3b188" }, { "path": "docs/benchmarks/raw/2026-08-13/concurrency-optimized-run-1.json", "sha256": "44ddaf62d43a3a73c40aca348aff175ec4e23ae01e75aa4be554650564bc8e35" }, { "path": "docs/benchmarks/raw/2026-08-13/concurrency-optimized-run-2.json", "sha256": "bb8e12b0d3a8aa56bdc0af3903a6fdd74e134440295244369a777cc417b544db" }, { "path": "docs/benchmarks/raw/2026-08-13/concurrency-optimized-run-3.json", "sha256": "6f9e0018844606136ec55b89d3da7ecb857bb4bb21f9cbc891a1b7a6456ac37a" }, { "path": "docs/benchmarks/raw/2026-08-13/neural-synthetic.json", "sha256": "251996d01847f20f4ac17ae964c4b9ca72e27e0a89908407c6761c9a88b2bb79" }, { "path": "docs/benchmarks/raw/2026-08-13/qemu-command-audit.json", "sha256": "374182fd18f13d3184e89c6ed4297aa54748ea4fb7b6495886bcb57c434830c2" }, { "path": "docs/benchmarks/raw/2026-08-14/cyber-physical-software.json", "sha256": "ef32ea6a79fe4945af7c5797d0eae743b0510c5206f80c1324e29ff8e6447530" }, { "path": "docs/research/world_model_paper_evaluation.json", "sha256": "59e79803e6d3430fc3e6694cbf72ca7acb19f90b8cb5c335692ace7748ae6d58" }, { "path": "docs/research/world_model_training_config.json", "sha256": "a6d0f3f198f28476c314951424de0114094fc8afc744c50b737c4d9aaed3604c" }, { "path": "docs/research/physical_world_model_evaluation.json", "sha256": "a7c9e36f846750ca6988c7434fa4c34baffe35567eb68ab930534a8d6c6b7111" } ] }, "qemu_command_audit": { "audit_source_commit": "45e369cececfc81e341c8772e295c4a00cba4bf1", "catalog_entries": 81, "catalog_passed": 81, "claim_boundary": "This is a dated emulator command-path audit for the named source commit and artifact hashes. It is not evidence of broad physical-PC compatibility, hardware accuracy, live EEG performance, formal safety, or independent replication.", "command_sweep_cases": 101, "command_sweep_passed": 101, "evidence_grade": "dated-emulator-measurement", "os_source_commit": "c92056d8635af5e4ee2a81351350b1b25cfd4861", "protocol_id": "ferrumos-qemu-command-audit-v1", "unknown_command_paths": 0 }, "schema_version": "2.1.0", "snapshot_date": "2026-08-14" }, "globalLimitations": [ "The paper, physical simulator, neural synthetic, QEMU, and cyber-physical software evaluations use different protocols and are not directly comparable.", "No live EEG, human neural calibration, robot hardware-in-the-loop, or broad physical-PC benchmark is claimed.", "Ring-3 timing uses a virtualized 1 kHz PIT; provider, tool execution, and operator-confirmation latency are excluded.", "A passing benchmark is evidence for its named fixture and protocol, not formal safety proof." ] }