{
  "$schema": "https://lumicorp.github.io/ruhroh/schemas/benchmark-claim-v1.schema.json",
  "version": "ruhroh_benchmark_claim_v1",
  "createdAt": "2026-07-08T12:00:00.000Z",
  "tool": {
    "name": "@kestrel-agents/ruhroh",
    "version": "0.6.0-beta.0"
  },
  "source": {
    "resultsPath": "sources/results",
    "suitePath": "sources/suite/suite.json",
    "suiteSha256": "77d974f71e63ae7bf435f9796a8c7a8dbc282f6bbaa2fd673502791c62b190fc",
    "runPlanPath": "sources/ruhroh-run-plan.json",
    "runPlanSha256": "f51a0b70f78b792eda39a5b6ef6a3717c6eaa474e7e909eaea126fdb972e437c",
    "htmlPath": "ruhroh-compare.html",
    "benchmarkClaimPath": "benchmark-claim.json",
    "benchmarkSummaryPath": "benchmark-summary.json",
    "resultArtifacts": [
      {
        "path": "sources/results/run-1/ruhroh-loop-result.json",
        "sha256": "19aafd3de86aea748b915e7bbdffcba7ef769873a843baca2a960c6cd195a568",
        "scenarioId": "simple-newsletter",
        "adapter": "agent-a",
        "runId": "sample-newsletter-agent-a-1",
        "sampleId": "simple-newsletter/agent-a/1-of-2",
        "scenarioVersion": "1.0.0",
        "benchmarkTarget": {
          "targetId": "docs-fixture-native-stack",
          "stream": "native-stack",
          "adapterId": "agent-a",
          "harness": {
            "name": "fixture-newsletter",
            "version": "docs-sample-v1"
          },
          "requestedModel": {
            "provider": "docs",
            "model": "sample-agent",
            "canonicalId": "docs/sample-agent",
            "version": "2026-07-08",
            "protocol": "fixture-command",
            "promptVersion": "docs-v1"
          },
          "actualModel": {
            "provider": "docs",
            "model": "sample-agent",
            "canonicalId": "docs/sample-agent",
            "version": "2026-07-08",
            "protocol": "fixture-command",
            "promptVersion": "docs-v1"
          },
          "providerPath": {
            "provider": "docs",
            "protocol": "fixture-command"
          },
          "recommendedStack": {
            "recommended": true,
            "rationale": "Docs sample uses the maintained fixture adapter with its native fixture model."
          }
        },
        "artifactInventory": [
          {
            "name": "evalInput",
            "path": "sources/results/run-1/ruhroh-loop-eval-input.json",
            "available": true,
            "sizeBytes": 122,
            "sha256": "3763a8caaa6ef3a89cdbc9aab8335e5aab51837d717e007ac6bdb83676239c1e"
          },
          {
            "name": "evalResult",
            "path": "sources/results/run-1/ruhroh-loop-eval.json",
            "available": true,
            "sizeBytes": 2465,
            "sha256": "076c551c4114b48d017aebf98480d339dc92b72ca3a4fb164a2f4074fc6eea93"
          },
          {
            "name": "events",
            "path": "sources/results/run-1/events.jsonl",
            "available": true,
            "sizeBytes": 57,
            "sha256": "354f962620ba81e40eee8813d3ad33cddbd01e7c40f9bf10166f843d7f7c7060"
          },
          {
            "name": "eventsTarball",
            "path": "sources/results/run-1/ruhroh-loop-events.tar.gz",
            "available": true,
            "sizeBytes": 53,
            "sha256": "2002b38a24ad7ff391d9b87433e1df4e852ebc937edf13e5bd95b44798e8b1e5"
          },
          {
            "name": "implementationRuns",
            "path": "sources/results/run-1/ruhroh-loop-iterations.jsonl",
            "available": true,
            "sizeBytes": 403,
            "sha256": "791fecab95e947282d21a5293f8b1a127310161277ac9efba6c50bc7d257d867"
          },
          {
            "name": "journey",
            "path": "sources/results/run-1/ruhroh-loop-journey.json",
            "available": true,
            "sizeBytes": 179,
            "sha256": "cd414b655de7de51f329079a8f9f8caaf6f4b417b2e99180e43acefa61d2e2f2"
          },
          {
            "name": "result",
            "path": "sources/results/run-1/ruhroh-loop-result.json",
            "available": true,
            "sizeBytes": 8393,
            "sha256": "19aafd3de86aea748b915e7bbdffcba7ef769873a843baca2a960c6cd195a568"
          },
          {
            "name": "runManifest",
            "path": "sources/results/run-1/ruhroh-run-manifest.json",
            "available": true,
            "sizeBytes": 2883,
            "sha256": "ddb29446d073b45540bb387b8bd8d751938580e5cdacce6f4c82ca692492eedc"
          },
          {
            "name": "transcript",
            "path": "sources/results/run-1/transcript.log",
            "available": true,
            "sizeBytes": 69,
            "sha256": "b64339f315ee62b69bca610f6689b7ba7e5172c6b15147fbed0ad2c28f02139a"
          },
          {
            "name": "transcriptsTarball",
            "path": "sources/results/run-1/ruhroh-loop-transcripts.tar.gz",
            "available": true,
            "sizeBytes": 58,
            "sha256": "454c0113d5058433b39a1ae789695420ce3cf42c8706ddc22778b671e55a8146"
          },
          {
            "name": "workspaceSummary",
            "path": "sources/results/run-1/ruhroh-workspace-summary.json",
            "available": true,
            "sizeBytes": 674,
            "sha256": "c5bcf5f724511667836f9a56cf409403be4e939d114d69265a4c395295f6cb29"
          },
          {
            "name": "workspaceTarball",
            "path": "sources/results/run-1/ruhroh-workspace.tar.gz",
            "available": true,
            "sizeBytes": 57,
            "sha256": "3c3f9efb11335aa9ef5106a20811c02a379548e6f7c7cedb4ccede370c2a1ca8"
          }
        ],
        "sizeBytes": 8393
      },
      {
        "path": "sources/results/run-2/ruhroh-loop-result.json",
        "sha256": "db0fd51fa1f598a701be9d2fee78c1f21a63b5ef6b6319a48c01a7fefced4bb0",
        "scenarioId": "simple-newsletter",
        "adapter": "agent-a",
        "runId": "sample-newsletter-agent-a-2",
        "sampleId": "simple-newsletter/agent-a/2-of-2",
        "scenarioVersion": "1.0.0",
        "benchmarkTarget": {
          "targetId": "docs-fixture-native-stack",
          "stream": "native-stack",
          "adapterId": "agent-a",
          "harness": {
            "name": "fixture-newsletter",
            "version": "docs-sample-v1"
          },
          "requestedModel": {
            "provider": "docs",
            "model": "sample-agent",
            "canonicalId": "docs/sample-agent",
            "version": "2026-07-08",
            "protocol": "fixture-command",
            "promptVersion": "docs-v1"
          },
          "actualModel": {
            "provider": "docs",
            "model": "sample-agent",
            "canonicalId": "docs/sample-agent",
            "version": "2026-07-08",
            "protocol": "fixture-command",
            "promptVersion": "docs-v1"
          },
          "providerPath": {
            "provider": "docs",
            "protocol": "fixture-command"
          },
          "recommendedStack": {
            "recommended": true,
            "rationale": "Docs sample uses the maintained fixture adapter with its native fixture model."
          }
        },
        "artifactInventory": [
          {
            "name": "evalInput",
            "path": "sources/results/run-2/ruhroh-loop-eval-input.json",
            "available": true,
            "sizeBytes": 122,
            "sha256": "3763a8caaa6ef3a89cdbc9aab8335e5aab51837d717e007ac6bdb83676239c1e"
          },
          {
            "name": "evalResult",
            "path": "sources/results/run-2/ruhroh-loop-eval.json",
            "available": true,
            "sizeBytes": 2645,
            "sha256": "d319f036ff5f3331b31819d61f5f182b4bf8943e542523e9962b847a906e9fa8"
          },
          {
            "name": "events",
            "path": "sources/results/run-2/events.jsonl",
            "available": true,
            "sizeBytes": 57,
            "sha256": "c866b4a17696f13dbce7e7b63e649de021acbf7e9cd6a55e26b3c44240de8fc9"
          },
          {
            "name": "eventsTarball",
            "path": "sources/results/run-2/ruhroh-loop-events.tar.gz",
            "available": true,
            "sizeBytes": 53,
            "sha256": "3f5e5b3c5e59edbefa9782469149a61fc6eb18b9eebdff1f990b614013b24f37"
          },
          {
            "name": "implementationRuns",
            "path": "sources/results/run-2/ruhroh-loop-iterations.jsonl",
            "available": true,
            "sizeBytes": 409,
            "sha256": "3d9b3814142e91963857753f9de32e03cac88db3d54ce487446ffda00b8cfd15"
          },
          {
            "name": "journey",
            "path": "sources/results/run-2/ruhroh-loop-journey.json",
            "available": true,
            "sizeBytes": 179,
            "sha256": "ede885c0cdce22f1b632cf6ac4d2b46beaedb1a1aa745ef8b622dcc667094609"
          },
          {
            "name": "result",
            "path": "sources/results/run-2/ruhroh-loop-result.json",
            "available": true,
            "sizeBytes": 8598,
            "sha256": "db0fd51fa1f598a701be9d2fee78c1f21a63b5ef6b6319a48c01a7fefced4bb0"
          },
          {
            "name": "runManifest",
            "path": "sources/results/run-2/ruhroh-run-manifest.json",
            "available": true,
            "sizeBytes": 2883,
            "sha256": "42c07b94425dd558bf6ffe6f6f99ade0a59868f72c0df9a502966b5633ce86df"
          },
          {
            "name": "transcript",
            "path": "sources/results/run-2/transcript.log",
            "available": true,
            "sizeBytes": 69,
            "sha256": "fc147d4635e80448bfd305b3aec71b5f0514de1c27b64057ab044fd3b03d3ed0"
          },
          {
            "name": "transcriptsTarball",
            "path": "sources/results/run-2/ruhroh-loop-transcripts.tar.gz",
            "available": true,
            "sizeBytes": 58,
            "sha256": "68eeb595f2849fd813675e5846e2bbb76cdf61566a8c7b6c953005fbea4aa4e2"
          },
          {
            "name": "workspaceSummary",
            "path": "sources/results/run-2/ruhroh-workspace-summary.json",
            "available": true,
            "sizeBytes": 674,
            "sha256": "c5bcf5f724511667836f9a56cf409403be4e939d114d69265a4c395295f6cb29"
          },
          {
            "name": "workspaceTarball",
            "path": "sources/results/run-2/ruhroh-workspace.tar.gz",
            "available": true,
            "sizeBytes": 57,
            "sha256": "39f79c358adad823b4d80be383fcadb1187f09bdd8f92722d30ca8eab5e59ff9"
          }
        ],
        "sizeBytes": 8598
      }
    ],
    "evaluatorCalibrationReportPath": "sources/evaluator-calibration/ruhroh-evaluator-calibration-report.json",
    "evaluatorCalibrationReportSha256": "02e73d36cb15cbfd897933740322f031a26c06d1d92930a408b08654e89a9d6a"
  },
  "scope": "suite",
  "publishable": false,
  "suite": {
    "id": "ruhroh-sample",
    "title": "Ruhroh Sample Evidence Pack",
    "suiteVersion": "0.1.0",
    "scenarioIds": [
      "simple-newsletter"
    ],
    "scenarioVersions": {
      "simple-newsletter": "1.0.0"
    },
    "minRuns": 5,
    "retryPolicy": "Docs sample only; real claims should collect the full planned run set."
  },
  "methodology": {
    "confidenceLevel": 0.95,
    "statisticalMethods": [
      "wilson_pass_rate_ci",
      "normal_approximation_pass_rate_delta_ci",
      "fisher_exact_two_sided",
      "pass_at_k",
      "bootstrap_mean_score_ci"
    ],
    "minRuns": 5,
    "retryPolicy": "Docs sample only; real claims should collect the full planned run set."
  },
  "summary": {
    "scenarioCount": 1,
    "adapterCount": 1,
    "totalRuns": 2,
    "totalPasses": 1,
    "runWeightedPassRate": 0.5,
    "runWeightedPassRateCi95": {
      "method": "wilson",
      "confidenceLevel": 0.95,
      "lower": 0.09453120573423071,
      "upper": 0.9054687942657693
    },
    "reviewRequired": 0,
    "reviewRecommended": 1,
    "pairwiseComparisonCount": 0
  },
  "adapterSummaries": [
    {
      "adapter": "agent-a",
      "scenarioCount": 1,
      "runs": 2,
      "passes": 1,
      "runWeightedPassRate": 0.5,
      "runWeightedPassRateCi95": {
        "method": "wilson",
        "confidenceLevel": 0.95,
        "lower": 0.09453120573423071,
        "upper": 0.9054687942657693
      },
      "meanScenarioPassRate": 0.5,
      "usage": {
        "runsWithUsage": 2,
        "runsWithCost": 2,
        "runsWithTokens": 2,
        "totalCostUsd": 0.05,
        "meanCostUsd": 0.025,
        "costPerPass": 0.05,
        "totalTokens": 1600,
        "meanTotalTokens": 800,
        "tokensPerPass": 1600
      },
      "minRunsSatisfied": false,
      "warnings": [
        "simple-newsletter has 2/5 required runs",
        "scenario-level warnings present"
      ]
    }
  ],
  "suiteCoverage": {
    "expectedScenarios": 1,
    "coveredScenarios": 1,
    "missingScenarioIds": [],
    "minRunsSatisfied": false,
    "adapters": [
      {
        "adapter": "agent-a",
        "expectedScenarios": 1,
        "coveredScenarios": 1,
        "missingScenarioIds": [],
        "scenarioRuns": {
          "simple-newsletter": 2
        },
        "minRunsSatisfied": false,
        "warnings": [
          "simple-newsletter has 2/5 required runs",
          "scenario-level warnings present"
        ]
      }
    ]
  },
  "scenarioResults": [
    {
      "scenarioId": "simple-newsletter",
      "adapter": "agent-a",
      "runs": 2,
      "passes": 1,
      "passRate": 0.5,
      "passRateCi95": {
        "method": "wilson",
        "confidenceLevel": 0.95,
        "lower": 0.09453120573423071,
        "upper": 0.9054687942657693
      },
      "passAtK": {
        "pass@1": 0.5,
        "pass@2": 1
      },
      "meanScore": 0.5,
      "meanScoreCi95": {
        "method": "bootstrap_percentile",
        "confidenceLevel": 0.95,
        "lower": 0,
        "upper": 1
      },
      "usage": {
        "runsWithUsage": 2,
        "runsWithCost": 2,
        "runsWithTokens": 2,
        "totalCostUsd": 0.05,
        "meanCostUsd": 0.025,
        "costPerPass": 0.05,
        "totalTokens": 1600,
        "meanTotalTokens": 800,
        "tokensPerPass": 1600
      },
      "reviewRequired": 0,
      "statisticalWarnings": [
        "fewer than 5 runs; treat pass rate and pass@k as directional"
      ],
      "cohort": {
        "sampleIds": [
          "simple-newsletter/agent-a/1-of-2",
          "simple-newsletter/agent-a/2-of-2"
        ],
        "sampleSeeds": [
          "docs-seed-one",
          "docs-seed-two"
        ],
        "scenarioVersions": [
          "1.0.0"
        ],
        "adapterVersions": [
          "docs-sample-v1"
        ],
        "benchmarkStreams": [
          "native-stack"
        ],
        "benchmarkTargets": [
          "docs-fixture-native-stack"
        ],
        "harnesses": [
          "fixture-newsletter@docs-sample-v1"
        ],
        "providerPaths": [
          "docs/fixture-command"
        ],
        "agentModels": [
          "docs/sample-agent@2026-07-08"
        ],
        "agentCanonicalModels": [
          "docs/sample-agent@2026-07-08"
        ],
        "agentPromptVersions": [
          "docs-v1"
        ],
        "evaluatorModels": [
          "docs/docs-fixture-evaluator@2026-07-08"
        ],
        "evaluatorPromptVersions": [
          "docs-eval-v1"
        ],
        "evaluatorInputSignatures": [
          "context=unknown;rubric=unknown;evidence=unknown;calibration=unknown;privateAssets=unknown;privateAssetPathHashes=none"
        ],
        "judgeIdentities": [
          "hybrid/docs-fixture-evaluator@2026-07-08"
        ],
        "environmentFingerprints": [
          "sha256:1111111111111111111111111111111111111111111111111111111111111111"
        ],
        "comparabilityWarnings": []
      }
    }
  ],
  "pairwiseComparisons": [],
  "readiness": {
    "scope": "suite",
    "publishable": false,
    "blockers": [
      "agent-a: suite minimum runs or scenario coverage not satisfied",
      "simple-newsletter/agent-a: fewer than 5 runs; treat pass rate and pass@k as directional",
      "simple-newsletter/agent-a: eval-quality warnings present"
    ],
    "advisories": [
      "agent-a: simple-newsletter has 2/5 required runs",
      "agent-a: scenario-level warnings present",
      "simple-newsletter/agent-a: eval command evidence includes non-zero exit codes (1)",
      "simple-newsletter/agent-a/sample-newsletter-agent-a-2: review recommended"
    ]
  },
  "evidence": {
    "runPlanPresent": true,
    "runPlanWarnings": [],
    "artifactValidationErrors": 0,
    "artifactValidationWarnings": 0,
    "artifactCompletenessWarnings": 0,
    "reviewQueueItems": 1,
    "requiredReviewItems": 0,
    "recommendedReviewItems": 1
  }
}