{ "schemaVersion": 1, "kind": "scorecard-details", "publicationId": "2026-10-27-windows-gemma4-12b-native-0-1-54", "complete": true, "sourceGeneratedAt": "2026-10-28T20:52:18.859Z", "sourceReportSha256": "f0163178da0cff20aaa89523348cb4b3e230840d8733dffefc7b2aaad0388120", "newInferenceCalls": 0, "counts": { "planned": 129, "recorded": 129, "rawPasses": 89, "capabilityExclusions": 13, "judged": 59, "notApplicable": 70, "noArtifact": 0, "excludedReviews": 3, "performanceRecords": 129, "pending": 0 }, "provenance": { "startedAt": "2026-10-28T07:03:41.388Z", "device": { "label": "win32 · AMD Ryzen AI 9 HX 370 w/ Radeon 890M", "platform": "win32", "arch": "x64", "memoryGb": 31, "osRelease": "win32 10.0.26300", "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M", "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU" }, "harnessCommit": "22fec99111a5ad9796af83a4a1a222236be91006", "sourceHash": "10a41de112c12aa33f12715ea0e1f30666cce2a8bf657064944820ed9f92b2e4", "serviceBuildHash": "8902bc3d7cf1ce309fbe752cdbff9a9a3dea0b27ef589f884c07c41e21602ace", "catalogHash": "2f12c2abbb9c4b216cda840c89d5683e7798b35e27fd683ea1f9bd864aa93e19", "gildeVersion": "0.3.1", "count": 3, "judgeModelId": "codex-inline/current-session", "engineBuilds": { "llama-cpp": "CUDA 12.8 / 620d2225cc436ab0b001e3200e590e3b286ef9ab / native 0.1.54" }, "runtimeHash": "21471af983e7130960ad0361be6f2a5c02d47cac0de9a3758083fa4986f40a42", "native": { "release": "0.1.54", "manifestSha256": "fe6fc6591d773499c17ad9431395cf62ecd9dd7846f1650c03a2f5417052f144", "cudaBuild": { "schemaVersion": 1, "engine": "llama-cpp", "revision": "620d2225cc436ab0b001e3200e590e3b286ef9ab", "platform": "win32-x64", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "cudaToolkit": "12.8" }, "version": "" }, "model": { "manifest": { "id": "gemma4-12b-q4", "name": "Gemma 4 (12B, Q4)", "approxSizeBytes": 6970065600, "weightsFilename": "gemma-4-12B-it-qat-UD-Q4_K_XL.gguf", "sha256": "90fd44e29e0d7cffeb0fd00dc73cfdab9ed0b0e95306ecf7821ea634c940c370", "installedAt": "2026-09-02T18:55:17.788Z", "catalogId": "gemma4-12b-q4", "catalogVersion": "1.2.0", "huggingfaceRepo": "unsloth/gemma-4-12B-it-qat-GGUF", "draftModelFilename": "mtp-gemma-4-12B-it-Q4_0.gguf", "quantization": "UD-Q4_K_XL", "ggufQuantization": "Q4_0", "contextWindow": 262144, "architecture": "gemma4", "chatTemplatePresent": true, "payloadFingerprint": "3ea5b73c22e9fcdd2047dd36df38aec903131c2e5fba67ef442abd1f4984a901", "fileSha256": { "gemma-4-12B-it-qat-UD-Q4_K_XL.gguf": "90fd44e29e0d7cffeb0fd00dc73cfdab9ed0b0e95306ecf7821ea634c940c370", "mtp-gemma-4-12B-it-Q4_0.gguf": "fcb35dea42c71333db904cee11baac525c9ef872818ee3753f6cb156f3c6f4f6" } }, "verifiedFileSha256": { "gemma-4-12B-it-qat-UD-Q4_K_XL.gguf": "90fd44e29e0d7cffeb0fd00dc73cfdab9ed0b0e95306ecf7821ea634c940c370", "mtp-gemma-4-12B-it-Q4_0.gguf": "fcb35dea42c71333db904cee11baac525c9ef872818ee3753f6cb156f3c6f4f6" } } }, "judging": { "provider": "codex-inline", "model": "current-session", "exactModelVersion": null, "inputLimitBytes": 20480, "blinding": "Unblinded artifact/source review; advisory, with scenario rubrics kept separate.", "note": "Original judgment text is preserved, including comments written before deferred browser validation. Final runtime outcomes appear separately.", "scenarioRubrics": [ { "suiteId": "core", "scenarioId": "tictactoe", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "petshop", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "tankcombat", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "schema-migration", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "failing-tests-spec", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "symptom-debug", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "data-wrangle", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "incident-postmortem", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "ops-runbook-anomaly", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "plan-and-estimate", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "core", "scenarioId": "conflict-synthesis", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "productivity", "scenarioId": "constrained-comms", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "productivity", "scenarioId": "records-intake", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "productivity", "scenarioId": "plan-and-estimate", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "productivity", "scenarioId": "meeting-followup", "judge": { "artifactBasename": "meeting-brief.md", "artifactKind": "markdown", "contextNote": "The August 27 transcript is authoritative; old-agenda.md contains rejected proposals that must not become decisions.", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ] }, "fallback": null }, { "suiteId": "productivity", "scenarioId": "craftbook-spreadsheet-model", "judge": { "artifactBasename": "scope.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." }, "fallback": null }, { "suiteId": "productivity", "scenarioId": "conflict-synthesis", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "judge": { "artifactBasename": "theme-report.md", "artifactKind": "markdown", "contextNote": "Theme inference returned exactly these values: theme id custom-brand-reference-pptx; heading font Kelder Grotesk; body font Marne Text; primary #2E5E4E; background #F4F1EA. The slide count and comparison figures depend on the session's own conversion, and a deterministic gate already checks them against the saved deck and the comparison result. Judge what it cannot: whether the report adds detail no tool returned, states the differences plainly, and ends somewhere useful. Do not mark a report down for lacking per-fact tool attribution — the brief does not ask for it.", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ] }, "fallback": null }, { "suiteId": "productivity", "scenarioId": "craftbook-powerpoint-deck", "judge": { "artifactBasename": "deck.md", "artifactKind": "markdown", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ] }, "fallback": null }, { "suiteId": "productivity", "scenarioId": "wikipedia-research-brief", "judge": { "artifactBasename": "research-brief.md", "artifactKind": "markdown", "contextNote": "All historical evidence comes from the three signed local Wikipedia source cards. The “first programmer” label must remain calibrated rather than asserted as uncontested fact.", "axes": [ { "name": "synthesis", "description": "Builds an argument across sources rather than listing source summaries." }, { "name": "grounding", "description": "Claims and dates stay faithful to the signed local source cards." }, { "name": "citationQuality", "description": "Source ids appear near the claims they support and make provenance easy to audit." }, { "name": "calibration", "description": "Separates documented facts, interpretation, and disputed historical shorthand." } ] }, "fallback": null }, { "suiteId": "productivity", "scenarioId": "craftbook-week-plan", "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ] }, "fallback": null }, { "suiteId": "productivity", "scenarioId": "craftbook-ab-test-readout", "judge": { "artifactBasename": "readout.md", "artifactKind": "markdown", "axes": [ { "name": "safety", "description": "Dry-run posture is explicit: no real credentials, clear stop-before-side-effect boundary." }, { "name": "mapping", "description": "Fake endpoints/commands are mapped completely to the workflow steps." }, { "name": "signals", "description": "Success, failure, retry, and abort conditions are all defined." }, { "name": "containment", "description": "The plan cannot accidentally reach a real service from a mock context." } ] }, "fallback": null }, { "suiteId": "productivity", "scenarioId": "craftbook-annotated-bibliography", "judge": { "artifactBasename": "bibliography.md", "artifactKind": "markdown", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ] }, "fallback": null }, { "suiteId": "productivity", "scenarioId": "craftbook-research-to-document", "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ] }, "fallback": null }, { "suiteId": "developer", "scenarioId": "fix-squisq-bugs", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "developer", "scenarioId": "dev-craftbook-routing", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "developer", "scenarioId": "interface-contract", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "developer", "scenarioId": "craftbook-code-review", "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." }, "fallback": null }, { "suiteId": "developer", "scenarioId": "craftbook-api-contract-review", "judge": { "artifactBasename": "api-review.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." }, "fallback": null }, { "suiteId": "developer", "scenarioId": "craftbook-deep-security-review", "judge": { "artifactBasename": "REPORT.md", "artifactKind": "markdown", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ] }, "fallback": null }, { "suiteId": "developer", "scenarioId": "large-pr-review", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "judge": { "artifactBasename": "verification.md", "artifactKind": "markdown", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ] }, "fallback": null }, { "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "judge": { "artifactBasename": "change-notes.md", "artifactKind": "markdown", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ] }, "fallback": null }, { "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "judge": { "artifactBasename": "fix-notes.md", "artifactKind": "markdown", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ] }, "fallback": null }, { "suiteId": "complex-work", "scenarioId": "craftbook-find-vs-create", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "complex-work", "scenarioId": "craftbook-edit-midtask", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "complex-work", "scenarioId": "craftbook-invoice-run", "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ] }, "fallback": null }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-linear", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "complex-work", "scenarioId": "craftbook-route-multi", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "complex-work", "scenarioId": "craftbook-export-generalize", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-params", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-gate-script", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-fanout", "judge": null, "fallback": "First HTML artifact uses frozen default HTML axes; no HTML means no applicable qualitative rubric." } ] }, "performance": { "allTrials": { "count": 129, "totalInputTokens": 18196306, "totalOutputTokens": 1919829, "durationMs": { "count": 129, "mean": 376763.3333333333, "min": 29051, "median": 169601, "p95": 1367696.5999999999, "max": 4778004, "sampleStdDev": 600567.2620966018 }, "totalTrialDurationMs": 48602470, "nativeDecodeTokensPerSec": { "count": 129, "mean": 121.3308527131783, "min": 97.1, "median": 121.16, "p95": 140.706, "max": 148.97, "sampleStdDev": 12.264107121431612 }, "nativePrefillTokensPerSec": { "count": 129, "mean": 1813.3889922480614, "min": 1236.6, "median": 1845.7, "p95": 1993.404, "max": 2087.06, "sampleStdDev": 149.76537506549286 }, "pooledDecodeTokensPerSecApprox": 120.65275875348966, "pooledPrefillTokensPerSecApprox": 1701.6401102761201, "peakRssMb": 21991, "peakGpuMemoryMb": 10730, "peakGpuUtilPercent": 100, "gpuMemoryTotalMb": [ 12227 ], "peakSystemUsedMb": 31687, "pooledRateMethod": "Total native tokens divided by sum(tokens / rounded trial-native rate). Approximate aggregate native throughput; excludes scheduling, tools and idle time. Not end-to-end throughput." }, "bySuite": { "core": { "count": 33, "totalInputTokens": 1476831, "totalOutputTokens": 211471, "durationMs": { "count": 33, "mean": 118284.63636363637, "min": 29954, "median": 57117, "p95": 381619.1999999996, "max": 726372, "sampleStdDev": 148789.502268057 }, "totalTrialDurationMs": 3903393, "nativeDecodeTokensPerSec": { "count": 33, "mean": 125.17060606060609, "min": 99.26, "median": 126.21, "p95": 144.82399999999998, "max": 148.97, "sampleStdDev": 13.740606095482548 }, "nativePrefillTokensPerSec": { "count": 33, "mean": 1902.45, "min": 1646.2, "median": 1927.35, "p95": 2001.342, "max": 2022.87, "sampleStdDev": 75.37883634515724 }, "pooledDecodeTokensPerSecApprox": 130.63708752623066, "pooledPrefillTokensPerSecApprox": 1899.2045589082595, "peakRssMb": 21156, "peakGpuMemoryMb": 10648, "peakGpuUtilPercent": 100, "gpuMemoryTotalMb": [ 12227 ], "peakSystemUsedMb": 31505, "pooledRateMethod": "Total native tokens divided by sum(tokens / rounded trial-native rate). Approximate aggregate native throughput; excludes scheduling, tools and idle time. Not end-to-end throughput." }, "productivity": { "count": 39, "totalInputTokens": 3416277, "totalOutputTokens": 420961, "durationMs": { "count": 39, "mean": 305954.76923076925, "min": 29051, "median": 153932, "p95": 520657.10000000003, "max": 4778004, "sampleStdDev": 747928.5722068251 }, "totalTrialDurationMs": 11932236, "nativeDecodeTokensPerSec": { "count": 39, "mean": 123.43487179487177, "min": 100.95, "median": 122.51, "p95": 139.394, "max": 143.1, "sampleStdDev": 9.292466635255757 }, "nativePrefillTokensPerSec": { "count": 39, "mean": 1873.2105128205133, "min": 1733.81, "median": 1857.89, "p95": 2044.193, "max": 2087.06, "sampleStdDev": 90.66291346894529 }, "pooledDecodeTokensPerSecApprox": 120.24808069595889, "pooledPrefillTokensPerSecApprox": 1837.78893457064, "peakRssMb": 21590, "peakGpuMemoryMb": 10606, "peakGpuUtilPercent": 100, "gpuMemoryTotalMb": [ 12227 ], "peakSystemUsedMb": 31510, "pooledRateMethod": "Total native tokens divided by sum(tokens / rounded trial-native rate). Approximate aggregate native throughput; excludes scheduling, tools and idle time. Not end-to-end throughput." }, "developer": { "count": 30, "totalInputTokens": 9925143, "totalOutputTokens": 791309, "durationMs": { "count": 30, "mean": 618716.6666666666, "min": 49762, "median": 313359, "p95": 1666045.8499999996, "max": 2653762, "sampleStdDev": 639637.9425846735 }, "totalTrialDurationMs": 18561500, "nativeDecodeTokensPerSec": { "count": 30, "mean": 123.97166666666666, "min": 109.44, "median": 121.85, "p95": 137.013, "max": 139.43, "sampleStdDev": 8.837646778114221 }, "nativePrefillTokensPerSec": { "count": 30, "mean": 1741.376, "min": 1236.6, "median": 1738.525, "p95": 1932.082, "max": 1947.13, "sampleStdDev": 171.83187180978732 }, "pooledDecodeTokensPerSecApprox": 126.12948298255473, "pooledPrefillTokensPerSecApprox": 1636.742713138411, "peakRssMb": 21846, "peakGpuMemoryMb": 10730, "peakGpuUtilPercent": 100, "gpuMemoryTotalMb": [ 12227 ], "peakSystemUsedMb": 31515, "pooledRateMethod": "Total native tokens divided by sum(tokens / rounded trial-native rate). Approximate aggregate native throughput; excludes scheduling, tools and idle time. Not end-to-end throughput." }, "complex-work": { "count": 27, "totalInputTokens": 3378055, "totalOutputTokens": 496088, "durationMs": { "count": 27, "mean": 526123.7407407408, "min": 48152, "median": 389416, "p95": 1487984.1999999995, "max": 2585107, "sampleStdDev": 539864.8960816026 }, "totalTrialDurationMs": 14205341, "nativeDecodeTokensPerSec": { "count": 27, "mean": 110.66444444444443, "min": 97.1, "median": 108.41, "p95": 139.045, "max": 142.73, "sampleStdDev": 11.967835526087713 }, "nativePrefillTokensPerSec": { "count": 27, "mean": 1698.142222222222, "min": 1259.77, "median": 1731.36, "p95": 1868.306, "max": 1869.46, "sampleStdDev": 152.55026698242864 }, "pooledDecodeTokensPerSecApprox": 109.78560323337271, "pooledPrefillTokensPerSecApprox": 1695.0268353189356, "peakRssMb": 21991, "peakGpuMemoryMb": 10521, "peakGpuUtilPercent": 100, "gpuMemoryTotalMb": [ 12227 ], "peakSystemUsedMb": 31687, "pooledRateMethod": "Total native tokens divided by sum(tokens / rounded trial-native rate). Approximate aggregate native throughput; excludes scheduling, tools and idle time. Not end-to-end throughput." } }, "wallClockMs": 48952730, "notes": [ "Native engine-log token counters remain separate from provider counters.", "Throughput excludes scheduling and tools; wall time includes them.", "RSS is system memory; VRAM is measured separately. MB fields are MiB. All attempts, including excluded attempts, are retained." ] }, "runtimeValidation": { "schemaVersion": 1, "finishedAt": "2026-10-28T20:43:12.227Z", "browser": { "executable": "", "sha256": "addfa79abb060e1e514e155ed745d4bf96140bca402735958bb4e223aea0b98c", "version": "Google Chrome for Testing 153.0.8010.12", "versionExitCode": 0, "versionStderr": null }, "sourceChecks": [ { "path": "evals/src/html-validation.ts", "rawSha256": "f047dbac25aeb0540bc5633408b6fc83510f47e0ac7673e08680a7739d792619", "identitySha256": "3568d03c8322fb168f1ccf377894cabd8321bc020d477670c1916c8d45a91530", "matchesLaunch": true }, { "path": "evals/src/chromium.ts", "rawSha256": "00d4cdb8c156b117fb397c96809af946cf67378a96ba56e732cb42b568fcc2cc", "identitySha256": "61a23e80a2d5a60d86c20977bb202228733f0494b697d0b75f81abf19906c386", "matchesLaunch": true }, { "path": "evals/src/scenarios/tictactoe.ts", "rawSha256": "2d771b4740c813336f54ca734490e27f391a4c0979fc23fae61eed65fe40314d", "identitySha256": "374abebe45c33b0e9f864849a45b1fca97318e309f6b3b6c139bc4fb6d429f4b", "matchesLaunch": true }, { "path": "evals/src/scenarios/petshop.ts", "rawSha256": "c6ae728c51d8f80c1f90580ed22263acddc43170a7d6451da92607c170dccae2", "identitySha256": "87ebf75879f96370f543648718ba0379f120c301599fbec1432505768dbfd08a", "matchesLaunch": true }, { "path": "evals/src/scenarios/tankcombat.ts", "rawSha256": "9e5d0971ddadbfd23213c1fe820a48d3124bfeda3d57b2d3b4e7729d813b63c5", "identitySha256": "17cbedf9c024763addd9029f9e92a5d3ccbb14acf1bac85b608626acb6b903a8", "matchesLaunch": true }, { "path": "evals/src/scenarios/helpers.ts", "rawSha256": "3cffbe792154ecf035b101ed9fb808ba6892e680a5069305677cc37ba8c1d69c", "identitySha256": "3396ea00b45dc6fc4cb1fbc7ad7669b9152a6c393aa694c3e214f67b407aca8d", "matchesLaunch": true } ], "expected": 9, "completed": 9, "ran": 9, "passed": 9, "failed": 0, "unavailable": 0, "reports": [ { "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv", "path": "tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv.json", "runtimePassed": true }, { "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q", "path": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q.json", "runtimePassed": true }, { "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f", "path": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f.json", "runtimePassed": true }, { "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4", "path": "petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4.json", "runtimePassed": true }, { "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti", "path": "petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti.json", "runtimePassed": true }, { "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5", "path": "petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5.json", "runtimePassed": true }, { "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i", "path": "tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i.json", "runtimePassed": true }, { "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw", "path": "tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw.json", "runtimePassed": true }, { "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p", "path": "tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p.json", "runtimePassed": true } ] }, "capability": { "rawPasses": 89, "rawTrials": 129, "eligiblePasses": 89, "eligibleTrials": 116, "fixedMean": 7.882170542635656, "eligibleFixedMean": 8.185344827586205 }, "trials": [ { "suiteId": "core", "scenarioId": "tictactoe", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv", "runDir": "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv", "finishedAt": "2026-10-28T07:06:01.921Z", "rawSuccess": true, "rawReason": "this-work-e/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 49127, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 5, "judgeArtifactCandidates": [ "workspace/this-work-e/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 143.06, "nativePrefillTokensPerSec": 1966.73, "inputTokens": 13590, "outputTokens": 1823, "peakRssMb": 12476, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10647, "gpuMemoryTotalMb": 12227, "durationMs": 49127, "derived": { "durationMs": 41180, "meanTokensPerSec": 143.06, "genTokensPerSec": 143.06, "genTokens": 1823, "promptTokensPerSec": 1966.73, "peakRssMb": 12476, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6976, "output": 23 } } }, "usage": { "available": true, "totalInputTokens": 13590, "totalOutputTokens": 1823, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6976, "todayTokensCached": 6804, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6976, "totalTokensCached": 6804, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 102.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 102.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6976, "cachedInputTokens": 6804, "outputTokens": 23, "cost": 0, "durationMs": 15798, "outputTokensPerSec": 102.94902151634548, "at": "2026-10-28T07:05:34.000Z" }, "lastUpdated": "2026-10-28T07:05:34.000Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:05:56.936Z" } }, "source": "engine-log" }, "process": { "sampleCount": 8, "peakRssMb": 12476, "peakCpuPercent": 259.9, "firstAtMs": 0, "lastAtMs": 39381 }, "gpu": { "available": true, "sampleCount": 8, "peakUtilPercent": 99, "peakMemUsedMb": 10647, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 28968, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 134144, "totalTokens": 134144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv.json", "startedAt": "2026-10-28T07:05:12.794Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:13:30.113Z", "scenarioId": "tictactoe", "suiteId": "core", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt; no rendered visual inspection; runtime validation pending.", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv", "artifactPath": "workspace/this-work-e/index.html", "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 6, "codeQuality": 6, "polish": 3 }, "justification": "Complete readable source for alternating turns, all win lines, draw detection, locked finished games and reset. Plain centered gray grid with hover states; no winner highlighting or score tracking. Fixed 340px board can overflow narrow screens; empty cells lack accessible names and status lacks live announcements. Browser behavior is pending separate validation.", "artifactSha256": "4c540405bb191275fd03e1b5a769ca2f952bd461fee9538cd4d3680edd93814b", "inputBytes": 4685, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5 }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 28968, "totalMb": 31865 }, "evidenceHashes": { "result.json": "b000d7f49c200804c09bb26e95f3ce190b25b7f78f8c2c6a33bb7b6a2c53d02f", "score.json": "ee4fecf5d49f3027f3a1667f1bab100f0edf24b9fc054f528897db7a71973e6b", "metrics.json": "d74b1e9525538300384f0919c3cdb8fcada258545449b4f7def6aa0e850cb932", "codex-inline-judge.json": "c6776c3e430de19a7c00e30b1ad43c3e20f1d584d56ccba890849716f19cb671" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv", "scenarioId": "tictactoe", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 40632 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "this-work-e/workspace/index.html", "bytes": 4685, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 4685 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.006823194444444444 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 40632 }, { "source": "facts", "path": "outcome.durationMs", "value": 49127 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "tictactoe", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q", "runDir": "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q", "finishedAt": "2026-10-28T07:06:54.476Z", "rawSuccess": true, "rawReason": "tic-tac-toe-game/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 51350, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 5.25, "judgeArtifactCandidates": [ "workspace/tic-tac-toe-game/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 140.25, "nativePrefillTokensPerSec": 1862.61, "inputTokens": 13759, "outputTokens": 2160, "peakRssMb": 13104, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10647, "gpuMemoryTotalMb": 12227, "durationMs": 51350, "derived": { "durationMs": 45797, "meanTokensPerSec": 140.25, "genTokensPerSec": 140.25, "genTokens": 2160, "promptTokensPerSec": 1862.61, "peakRssMb": 13104, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7115, "output": 25 } } }, "usage": { "available": true, "totalInputTokens": 13759, "totalOutputTokens": 2160, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7115, "todayTokensCached": 6931, "todayTokensOut": 25, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7115, "totalTokensCached": 6931, "totalTokensOut": 25, "totalCost": 0, "medianOutputTokensPerSec": 101.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 101.2, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7115, "cachedInputTokens": 6931, "outputTokens": 25, "cost": 0, "durationMs": 17522, "outputTokensPerSec": 101.16295244076699, "at": "2026-10-28T07:06:26.342Z" }, "lastUpdated": "2026-10-28T07:06:26.342Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:06:52.192Z" } }, "source": "engine-log" }, "process": { "sampleCount": 9, "peakRssMb": 13104, "peakCpuPercent": 273.8, "firstAtMs": 0, "lastAtMs": 44711 }, "gpu": { "available": true, "sampleCount": 9, "peakUtilPercent": 99, "peakMemUsedMb": 10647, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 29398, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 134144, "totalTokens": 134144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q.json", "startedAt": "2026-10-28T07:06:03.126Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:13:30.115Z", "scenarioId": "tictactoe", "suiteId": "core", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt; no rendered visual inspection; runtime validation pending.", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q", "artifactPath": "workspace/tic-tac-toe-game/index.html", "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 6, "codeQuality": 6, "polish": 4 }, "justification": "Readable standalone game implements turn alternation, wins, draws, repeat-click guards and restart. Larger marks, hover transitions and reserved status height add modest finish. Styling remains generic; fixed board sizing, unlabeled empty cells and no live status weaken usability. Source was reviewed in full; browser behavior awaits separate validation.", "artifactSha256": "acbfb6db83b99860ef1280e38fdb61c35096f3c796cfa6e3492b2d86d118bed7", "inputBytes": 4993, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.25 }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 29398, "totalMb": 31865 }, "evidenceHashes": { "result.json": "86d931bc3a664283d9b14208f232fbf907bfac5171c922fa602378ba18d7ad71", "score.json": "017db071966144ec19bf27e79789704ace15f12d23ac3409adbaff50ec32081a", "metrics.json": "5b6fd7f663b40f8f98a5deb57fd7672e5f0532cec584556ca9c73a46ee6d4798", "codex-inline-judge.json": "e2909baed8dbf0cfd69a90676f135533452c12f0a2edf6a882d67ea16c508184" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q", "scenarioId": "tictactoe", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 43640 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 4993, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 4993 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.007131944444444444 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 43640 }, { "source": "facts", "path": "outcome.durationMs", "value": 51350 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "tictactoe", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f", "runDir": "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f", "finishedAt": "2026-10-28T07:07:44.308Z", "rawSuccess": true, "rawReason": "this-work-e/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 49390, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 5.25, "judgeArtifactCandidates": [ "workspace/this-work-e/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 137.58, "nativePrefillTokensPerSec": 1990.95, "inputTokens": 13536, "outputTokens": 1852, "peakRssMb": 12490, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10647, "gpuMemoryTotalMb": 12227, "durationMs": 49390, "derived": { "durationMs": 40191, "meanTokensPerSec": 137.58, "genTokensPerSec": 137.58, "genTokens": 1852, "promptTokensPerSec": 1990.95, "peakRssMb": 12490, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6922, "output": 23 } } }, "usage": { "available": true, "totalInputTokens": 13536, "totalOutputTokens": 1852, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6922, "todayTokensCached": 6750, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6922, "totalTokensCached": 6750, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 110.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 110.7, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6922, "cachedInputTokens": 6750, "outputTokens": 23, "cost": 0, "durationMs": 14927, "outputTokensPerSec": 110.73472489970958, "at": "2026-10-28T07:07:15.493Z" }, "lastUpdated": "2026-10-28T07:07:15.493Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:07:38.380Z" } }, "source": "engine-log" }, "process": { "sampleCount": 8, "peakRssMb": 12490, "peakCpuPercent": 271.3, "firstAtMs": 1, "lastAtMs": 39067 }, "gpu": { "available": true, "sampleCount": 8, "peakUtilPercent": 99, "peakMemUsedMb": 10647, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 28788, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 134144, "totalTokens": 134144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f.json", "startedAt": "2026-10-28T07:06:54.918Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:13:30.116Z", "scenarioId": "tictactoe", "suiteId": "core", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt; no rendered visual inspection; runtime validation pending.", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f", "artifactPath": "workspace/this-work-e/index.html", "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 6, "codeQuality": 6, "polish": 4 }, "justification": "Clear state and event handling cover alternating moves, winning lines, draw, lockout and complete reset. Blue X/red O marks and hover transitions improve legibility. Generic fixed-size layout lacks narrow-screen adaptation, accessible empty-cell names and a live status region. No winner highlight. Browser behavior remains pending independent validation.", "artifactSha256": "cd740b4d3bb35085bf35ea43b3e6dcb9dfec997de39ab25e67ff4796a0874082", "inputBytes": 4877, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.25 }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 28788, "totalMb": 31865 }, "evidenceHashes": { "result.json": "cf0d75e4c814050ce1e5ded8414034f121e85c30152f3b07d1bc34aaf9387442", "score.json": "272cfce4c10cbf81431056e3e90c0d337b4c66097d5877702d3adce91aa97c1d", "metrics.json": "d3e79e004590057c7eb199c20da0a3eef2032e5fbef4baeaa43286a385ea883b", "codex-inline-judge.json": "7fde6ebf2c0c23b69af6fa59024b74e88bdbd5d5554ba9dbf03d17ceb91e51e0" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f", "scenarioId": "tictactoe", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 42758 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "this-work-e/workspace/index.html", "bytes": 4877, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 4877 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.006859722222222222 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 42758 }, { "source": "facts", "path": "outcome.durationMs", "value": 49390 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "petshop", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4", "runDir": "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4", "finishedAt": "2026-10-28T07:11:10.258Z", "rawSuccess": true, "rawReason": "build-me-a-single-page-pet-shop-website/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 201856, "rawFixedComposite": 8, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 4.75, "judgeArtifactCandidates": [ "workspace/build-me-a-single-page-pet-shop-website/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 133.43, "nativePrefillTokensPerSec": 1857.08, "inputTokens": 81925, "outputTokens": 8321, "peakRssMb": 17876, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10647, "gpuMemoryTotalMb": 12227, "durationMs": 201856, "derived": { "durationMs": 188305, "meanTokensPerSec": 133.43, "genTokensPerSec": 133.43, "genTokens": 8321, "promptTokensPerSec": 1857.08, "peakRssMb": 17876, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 45977, "output": 1490 } } }, "usage": { "available": true, "totalInputTokens": 81925, "totalOutputTokens": 8321, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 45977, "todayTokensCached": 37790, "todayTokensOut": 1490, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 45977, "totalTokensCached": 37790, "totalTokensOut": 1490, "totalCost": 0, "medianOutputTokensPerSec": 99.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 99.6, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7335, "cachedInputTokens": 7137, "outputTokens": 29, "cost": 0, "durationMs": 21838, "outputTokensPerSec": 99.56546785102162, "at": "2026-10-28T07:10:26.845Z" }, "lastUpdated": "2026-10-28T07:10:26.845Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:11:00.008Z" } }, "source": "engine-log" }, "process": { "sampleCount": 34, "peakRssMb": 17876, "peakCpuPercent": 296.2, "firstAtMs": 0, "lastAtMs": 187228 }, "gpu": { "available": true, "sampleCount": 34, "peakUtilPercent": 100, "peakMemUsedMb": 10647, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 31505, "totalMb": 31865 } }, "engineContext": { "launches": 2, "grantedPerSlotTokens": 134144, "totalTokens": 134144, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4.json", "startedAt": "2026-10-28T07:07:48.402Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:22:34.417Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt. No browser rendering or image-pixel review. Runtime validation pending separately.", "suiteId": "core", "scenarioId": "petshop", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4", "artifactPath": "workspace/build-me-a-single-page-pet-shop-website/index.html", "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 5, "codeQuality": 5, "polish": 4 }, "justification": "Local PNG logo is wired into a complete landing page with working section targets, service descriptions and contact copy. CSS provides a coherent green/orange theme and sticky navigation. Hero depends on a remote photo; header has no mobile adaptation and 300px grid minimum can overflow narrow screens. Generic copy and low-contrast orange CTA limit polish. Source review only.", "artifactSha256": "061778a9c00ff40ee8e3f4bf2c683c8db8e6431280f52eeb8f23809ca6fb89d9", "inputBytes": 4534, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4.75 }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 31505, "totalMb": 31865 }, "evidenceHashes": { "result.json": "f50d2e3343add0ee16ddf15a8a3c4da6387fd93a44f364f499ba5fdb2eb4c5c2", "score.json": "87548da2afaa7a1d4a0641fdb66f0f213cc5a2d539a3b371c6108775e0b48aaf", "metrics.json": "e85f9500f5cce2b92b366db7044bf5bc30bc110a80ff786ba35567febcafc1b5", "codex-inline-judge.json": "01e09be83d255e99cac1626c0223446edf4e275b9ec7bd8d4c1a6f0876fec697" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4", "scenarioId": "petshop", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "requiredRasterMinBytes": 51200 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 66599 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "build-me-a-single-page-pet-shop-website/workspace/index.html", "bytes": 4534, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "artifacts.imageFiles", "value": [ { "path": "workspace/build-me-a-single-page-pet-shop-website/assets/logo.png", "bytes": 1033734 }, { "path": "artifacts/build-me-a-single-page-pet-shop-website/generated/image-2026-10-28T07-10-57-322Z-56115072.png", "bytes": 1033734 }, { "path": "artifacts/build-me-a-single-page-pet-shop-website/sessions/2026-10-28_070901_new-session/tool-0-img-0.png", "bytes": 1033734 } ] } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.028035555555555557 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 66599 }, { "source": "facts", "path": "outcome.durationMs", "value": 201856 } ] }, "behavior": { "score": 0, "ruleId": "B_RED_FLAG_OR_MISSING_ROLE", "summary": "At least one red-flag behavior fired or an expected role was missing.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [ "image-generator" ] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "petshop", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti", "runDir": "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti", "finishedAt": "2026-10-28T07:13:03.512Z", "rawSuccess": true, "rawReason": "build-me-a-single-page-pet-shop-website/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 112798, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 5, "judgeArtifactCandidates": [ "workspace/build-me-a-single-page-pet-shop-website/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 129.83, "nativePrefillTokensPerSec": 1848.1, "inputTokens": 37191, "outputTokens": 3321, "peakRssMb": 17258, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10648, "gpuMemoryTotalMb": 12227, "durationMs": 112798, "derived": { "durationMs": 100207, "meanTokensPerSec": 129.83, "genTokensPerSec": 129.83, "genTokens": 3321, "promptTokensPerSec": 1848.1, "peakRssMb": 17258, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 38652, "output": 123 } } }, "usage": { "available": true, "totalInputTokens": 37191, "totalOutputTokens": 3321, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 38652, "todayTokensCached": 38022, "todayTokensOut": 123, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 38652, "totalTokensCached": 38022, "totalTokensOut": 123, "totalCost": 0, "medianOutputTokensPerSec": 96.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 96.9, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16093, "cachedInputTokens": 15880, "outputTokens": 51, "cost": 0, "durationMs": 5954, "outputTokensPerSec": 96.85023660512803, "at": "2026-10-28T07:11:57.999Z" }, "lastUpdated": "2026-10-28T07:11:57.999Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:12:54.440Z" } }, "source": "engine-log" }, "process": { "sampleCount": 18, "peakRssMb": 17258, "peakCpuPercent": 262.6, "firstAtMs": 0, "lastAtMs": 99118 }, "gpu": { "available": true, "sampleCount": 18, "peakUtilPercent": 99, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 31467, "totalMb": 31865 } }, "engineContext": { "launches": 2, "grantedPerSlotTokens": 143360, "totalTokens": 143360, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti.json", "startedAt": "2026-10-28T07:11:10.714Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:22:34.424Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt. No browser rendering or image-pixel review. Runtime validation pending separately.", "suiteId": "core", "scenarioId": "petshop", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti", "artifactPath": "workspace/build-me-a-single-page-pet-shop-website/index.html", "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 6, "codeQuality": 5, "polish": 4 }, "justification": "Local logo, priced products, shop navigation and a simple cart/count/total exceed a bare landing page. Cohesive purple styling and grid cards are clear in source. Cart interaction relies on blocking alerts with no removal or persistence; hero photo is remote and header lacks a mobile layout. Product images are absent. No rendered or logo-pixel review.", "artifactSha256": "51fc230cd00d7c72a67c627b1d7022bdf09a4178b2c645a0fc450b40885e2f00", "inputBytes": 4859, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5 }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 31467, "totalMb": 31865 }, "evidenceHashes": { "result.json": "7ce5580b73214761e2a5879580fdc767b7a98b50d538d632e3c7a61ec80dfbd9", "score.json": "cb0bd38841c55f58a3b5364bfa40c58c053381573266ba41c6004fd2bba61c6d", "metrics.json": "7245d4ec0a4105f327fbaf1dc7cab5a4adf22f23139b49ec42508ba12a6f4a2c", "codex-inline-judge.json": "7c8ed5bc35e99366a7e93825ffa5f2aa9ad1c51549f290222ec8f566d3ad6829" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti", "scenarioId": "petshop", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "requiredRasterMinBytes": 51200 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 73618 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "build-me-a-single-page-pet-shop-website/workspace/index.html", "bytes": 4859, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "artifacts.imageFiles", "value": [ { "path": "workspace/build-me-a-single-page-pet-shop-website/assets/logo.png", "bytes": 1147924 }, { "path": "artifacts/build-me-a-single-page-pet-shop-website/generated/image-2026-10-28T07-12-53-354Z-1863508480.png", "bytes": 1147924 }, { "path": "artifacts/build-me-a-single-page-pet-shop-website/sessions/2026-10-28_071150_new-session/tool-0-img-0.png", "bytes": 1147924 } ] } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.01566638888888889 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 73618 }, { "source": "facts", "path": "outcome.durationMs", "value": 112798 } ] }, "behavior": { "score": 0, "ruleId": "B_RED_FLAG_OR_MISSING_ROLE", "summary": "At least one red-flag behavior fired or an expected role was missing.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [ "image-generator" ] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "petshop", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5", "runDir": "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5", "finishedAt": "2026-10-28T07:15:32.095Z", "rawSuccess": true, "rawReason": "build-me-a-single-page-pet-shop-website/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 147724, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 5.25, "judgeArtifactCandidates": [ "workspace/build-me-a-single-page-pet-shop-website/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 132.33, "nativePrefillTokensPerSec": 1885.46, "inputTokens": 45485, "outputTokens": 3979, "peakRssMb": 18710, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10603, "gpuMemoryTotalMb": 12227, "durationMs": 147724, "derived": { "durationMs": 136016, "meanTokensPerSec": 132.33, "genTokensPerSec": 132.33, "genTokens": 3979, "promptTokensPerSec": 1885.46, "peakRssMb": 18710, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 41680, "output": 141 } } }, "usage": { "available": true, "totalInputTokens": 45485, "totalOutputTokens": 3979, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 41680, "todayTokensCached": 40583, "todayTokensOut": 141, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 41680, "totalTokensCached": 40583, "totalTokensOut": 141, "totalCost": 0, "medianOutputTokensPerSec": 94.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 94.1, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8532, "cachedInputTokens": 8324, "outputTokens": 34, "cost": 0, "durationMs": 14010, "outputTokensPerSec": 88.75333833579423, "at": "2026-10-28T07:14:58.042Z" }, "lastUpdated": "2026-10-28T07:14:58.042Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:15:24.095Z" } }, "source": "engine-log" }, "process": { "sampleCount": 25, "peakRssMb": 18710, "peakCpuPercent": 267.2, "firstAtMs": 0, "lastAtMs": 134793 }, "gpu": { "available": true, "sampleCount": 25, "peakUtilPercent": 99, "peakMemUsedMb": 10603, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 29098, "totalMb": 31865 } }, "engineContext": { "launches": 2, "grantedPerSlotTokens": 149504, "totalTokens": 149504, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5.json", "startedAt": "2026-10-28T07:13:04.371Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:22:34.426Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt. No browser rendering or image-pixel review. Runtime validation pending separately.", "suiteId": "core", "scenarioId": "petshop", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5", "artifactPath": "workspace/build-me-a-single-page-pet-shop-website/index.html", "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 6, "codeQuality": 5, "polish": 5 }, "justification": "Local logo and five products form a complete single-page shop. On-page cart rows, count, formatted total and checkout/reset provide clearer feedback than alerts alone. Consistent purple cards and hover lift add finish, but checkout is simulated, items cannot be removed, and there are no product photos or contact details. Full source reviewed; rendering and logo appearance unverified.", "artifactSha256": "18882663535f9a694d9622e77c8b6c05e3c5d8cc2e769aa0969229402e29deb3", "inputBytes": 5861, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.25 }, "systemMemory": { "sampleCount": 25, "peakUsedMb": 29098, "totalMb": 31865 }, "evidenceHashes": { "result.json": "c674fa95b984b9c125327febe038d918d102a967a634cb00874b74c8e95ee7e3", "score.json": "9fcdba4df1eda5440c0ef2739ce02868c015e4d4b0934d9aa2215cb618ad18d8", "metrics.json": "85ecbff8e02eb8cb0ea5a64055837ec0514b7dce5e204d698d81e7cb09a765e1", "codex-inline-judge.json": "f298c6691892eab3c750b72d05e4d05e9c2b478290bf8f73b4ead4b1d631a859" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5", "scenarioId": "petshop", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "requiredRasterMinBytes": 51200 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 134548 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "build-me-a-single-page-pet-shop-website/workspace/index.html", "bytes": 5861, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "artifacts.imageFiles", "value": [ { "path": "workspace/build-me-a-single-page-pet-shop-website/assets/logo.png", "bytes": 969462 }, { "path": "artifacts/build-me-a-single-page-pet-shop-website/generated/image-2026-10-28T07-14-28-923Z-931928145.png", "bytes": 969462 }, { "path": "artifacts/build-me-a-single-page-pet-shop-website/sessions/2026-10-28_071351_new-session/tool-0-img-0.png", "bytes": 969462 } ] } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.02051722222222222 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 134548 }, { "source": "facts", "path": "outcome.durationMs", "value": 147724 } ] }, "behavior": { "score": 0, "ruleId": "B_RED_FLAG_OR_MISSING_ROLE", "summary": "At least one red-flag behavior fired or an expected role was missing.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [ "image-generator" ] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "tankcombat", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i", "runDir": "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i", "finishedAt": "2026-10-28T07:16:33.374Z", "rawSuccess": true, "rawReason": "this-work-e/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 57117, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 2.75, "judgeArtifactCandidates": [ "workspace/this-work-e/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 135.95, "nativePrefillTokensPerSec": 1888.96, "inputTokens": 13307, "outputTokens": 2980, "peakRssMb": 12615, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10603, "gpuMemoryTotalMb": 12227, "durationMs": 57117, "derived": { "durationMs": 51016, "meanTokensPerSec": 135.95, "genTokensPerSec": 135.95, "genTokens": 2980, "promptTokensPerSec": 1888.96, "peakRssMb": 12615, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7142, "output": 23 } } }, "usage": { "available": true, "totalInputTokens": 13307, "totalOutputTokens": 2980, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7142, "todayTokensCached": 6970, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7142, "totalTokensCached": 6970, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 81.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 81.4, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7142, "cachedInputTokens": 6970, "outputTokens": 23, "cost": 0, "durationMs": 18071, "outputTokensPerSec": 81.37960627066856, "at": "2026-10-28T07:16:00.073Z" }, "lastUpdated": "2026-10-28T07:16:00.073Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:16:30.641Z" } }, "source": "engine-log" }, "process": { "sampleCount": 10, "peakRssMb": 12615, "peakCpuPercent": 268.3, "firstAtMs": 0, "lastAtMs": 50211 }, "gpu": { "available": true, "sampleCount": 10, "peakUtilPercent": 99, "peakMemUsedMb": 10603, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 23208, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 149504, "totalTokens": 149504, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i.json", "startedAt": "2026-10-28T07:15:36.257Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:22:34.428Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt. No browser rendering or image-pixel review. Runtime validation pending separately.", "suiteId": "core", "scenarioId": "tankcombat", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i", "artifactPath": "workspace/this-work-e/index.html", "scoreAxes": { "visualQuality": 4, "functionalCompleteness": 2, "codeQuality": 3, "polish": 2 }, "justification": "Source provides tanks, movement, keyboard observability and a score display, but bullets spawn at the shooter's center and collision marks them inactive even against friendly tanks; after one 7px step they remain within the 24px hit radius. Normal shots are therefore consumed by their owner. Enemy never shoots, no defeat/restart UI, stale gameState.score and leftover placeholder AI.", "artifactSha256": "4302bc1de24baa53fa5fefe81b3db80b8df67a841595e9fa3a66db743d0e9a94", "inputBytes": 8126, "inputLimitBytes": 20480, "truncated": false, "meanScore": 2.75 }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 23208, "totalMb": 31865 }, "evidenceHashes": { "result.json": "8fb549eee3f046d6e979ada57df9ae7378890572392a7e19cd20188000183273", "score.json": "f5695dacaea5a44eae7a65e3df5c73538871ee589b0d34b75dbaa2bd17e55a73", "metrics.json": "34d136cfc667ec268d46e3c03612430f86c5ab69344c054f628926b523d479f4", "codex-inline-judge.json": "b10089ab9ba464c57633679183b5832abc6765871a918d0c7e4f1a300cde69ac" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i", "scenarioId": "tankcombat", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51193 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "this-work-e/workspace/index.html", "bytes": 8126, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 8126 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.007932916666666666 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51193 }, { "source": "facts", "path": "outcome.durationMs", "value": 57117 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "tankcombat", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw", "runDir": "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw", "finishedAt": "2026-10-28T07:17:30.874Z", "rawSuccess": true, "rawReason": "this-work-e/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 56874, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 4, "judgeArtifactCandidates": [ "workspace/this-work-e/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 129.87, "nativePrefillTokensPerSec": 1930.41, "inputTokens": 13506, "outputTokens": 3309, "peakRssMb": 12588, "gpuAvailable": true, "peakGpuUtilPercent": 85, "peakGpuMemoryMb": 10603, "gpuMemoryTotalMb": 12227, "durationMs": 56874, "derived": { "durationMs": 51143, "meanTokensPerSec": 129.87, "genTokensPerSec": 129.87, "genTokens": 3309, "promptTokensPerSec": 1930.41, "peakRssMb": 12588, "peakGpuUtilPercent": 85, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7343, "output": 19 } } }, "usage": { "available": true, "totalInputTokens": 13506, "totalOutputTokens": 3309, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7343, "todayTokensCached": 7174, "todayTokensOut": 19, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7343, "totalTokensCached": 7174, "totalTokensOut": 19, "totalCost": 0, "medianOutputTokensPerSec": 87.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 87.5, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7343, "cachedInputTokens": 7174, "outputTokens": 19, "cost": 0, "durationMs": 23663, "outputTokensPerSec": 87.49459234811181, "at": "2026-10-28T07:17:02.997Z" }, "lastUpdated": "2026-10-28T07:17:02.997Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:17:28.178Z" } }, "source": "engine-log" }, "process": { "sampleCount": 10, "peakRssMb": 12588, "peakCpuPercent": 263.7, "firstAtMs": 0, "lastAtMs": 50280 }, "gpu": { "available": true, "sampleCount": 10, "peakUtilPercent": 85, "peakMemUsedMb": 10603, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 24820, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 149504, "totalTokens": 149504, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw.json", "startedAt": "2026-10-28T07:16:34.000Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:22:34.430Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt. No browser rendering or image-pixel review. Runtime validation pending separately.", "suiteId": "core", "scenarioId": "tankcombat", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw", "artifactPath": "workspace/this-work-e/index.html", "scoreAxes": { "visualQuality": 4, "functionalCompleteness": 5, "codeQuality": 4, "polish": 3 }, "justification": "Source implements turning/movement, spaced shots, pursuing enemies, collision scoring and game over. Colored tank shapes and visible score are readable but sparse; controls are undocumented and defeat uses alert plus reload. Splicing enemies/bullets during forEach can skip entries; normalization lacks a zero-distance guard and movement is frame-dependent. Browser validation pending.", "artifactSha256": "8f6e1cfe7c5ed960d1dc21808676988134c161c4fc7b917003f0349ec8a1cd39", "inputBytes": 6786, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4 }, "systemMemory": { "sampleCount": 10, "peakUsedMb": 24820, "totalMb": 31865 }, "evidenceHashes": { "result.json": "fabc9b0bd7922a98c519a073e533e33e624eb565bb1e01e0affb34eaa1b031ce", "score.json": "b8019225cc7218aae62486c855111b357886d397e7923a406b302ac12cbf9cf0", "metrics.json": "657769f95747c2d9c579849a674776581b0df4aef2a0d95901daad3926bf2fb5", "codex-inline-judge.json": "e1539d83338190b85e15d43277591ac08b16ba5fed2e186c2e5fb12ebfb1af4a" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw", "scenarioId": "tankcombat", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 53410 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "this-work-e/workspace/index.html", "bytes": 6786, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 6786 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.007899166666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 53410 }, { "source": "facts", "path": "outcome.durationMs", "value": 56874 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "tankcombat", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p", "runDir": "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p", "finishedAt": "2026-10-28T07:18:22.851Z", "rawSuccess": true, "rawReason": "this-work-e/workspace/index.html passed sniff; runtime layer unavailable", "durationMs": 51310, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "posthoc-runtime-passed", "judgmentStatus": "judged", "judgmentMean": 3.5, "judgeArtifactCandidates": [ "workspace/this-work-e/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 133.35, "nativePrefillTokensPerSec": 1957.52, "inputTokens": 13528, "outputTokens": 2785, "peakRssMb": 12617, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10603, "gpuMemoryTotalMb": 12227, "durationMs": 51310, "derived": { "durationMs": 45489, "meanTokensPerSec": 133.35, "genTokensPerSec": 133.35, "genTokens": 2785, "promptTokensPerSec": 1957.52, "peakRssMb": 12617, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7363, "output": 18 } } }, "usage": { "available": true, "totalInputTokens": 13528, "totalOutputTokens": 2785, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7363, "todayTokensCached": 7191, "todayTokensOut": 18, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7363, "totalTokensCached": 7191, "totalTokensOut": 18, "totalCost": 0, "medianOutputTokensPerSec": 98.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 98.4, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7363, "cachedInputTokens": 7191, "outputTokens": 18, "cost": 0, "durationMs": 17285, "outputTokensPerSec": 98.42063822889166, "at": "2026-10-28T07:17:54.085Z" }, "lastUpdated": "2026-10-28T07:17:54.085Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:18:19.930Z" } }, "source": "engine-log" }, "process": { "sampleCount": 9, "peakRssMb": 12617, "peakCpuPercent": 275.9, "firstAtMs": 0, "lastAtMs": 44668 }, "gpu": { "available": true, "sampleCount": 9, "peakUtilPercent": 99, "peakMemUsedMb": 10603, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 23237, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 149504, "totalTokens": 149504, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "runtimeEvidence": "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p.json", "startedAt": "2026-10-28T07:17:31.541Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": { "ran": true, "passed": true, "failures": [], "evidence": "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p.json" }, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T07:22:34.432Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Complete HTML source and original scenario prompt. No browser rendering or image-pixel review. Runtime validation pending separately.", "suiteId": "core", "scenarioId": "tankcombat", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p", "artifactPath": "workspace/this-work-e/index.html", "scoreAxes": { "visualQuality": 4, "functionalCompleteness": 4, "codeQuality": 3, "polish": 3 }, "justification": "Acceleration/friction, pursuit, projectiles and score implement an arcade loop, with clear colored silhouettes. On player collision the backward enemy loop replaces enemies with an empty array without breaking; if i was above zero, the next iteration dereferences undefined. Reset also preserves projectiles and speed. No control help, damage feedback or restart screen. Source review only.", "artifactSha256": "2c9c1715b038c86ffb62ca69f60b9ce601dc7a20b50135efc81257a36fc075da", "inputBytes": 7301, "inputLimitBytes": 20480, "truncated": false, "meanScore": 3.5 }, "systemMemory": { "sampleCount": 9, "peakUsedMb": 23237, "totalMb": 31865 }, "evidenceHashes": { "result.json": "dffe3d86c9f10ae50c6a200a084ff72fd74d58aaa3d48672c3bbef276b7f5458", "score.json": "f5c8e41a7fb013a8c0264194746582dd7a90ccd130d457c71ecba0e3f7e85108", "metrics.json": "834b357d9ded43c4ff63e8f10592df56fb64e75be0a7d3a99cf7e85fe68ca348", "codex-inline-judge.json": "27db3dc8501981cba5bfc83265bf943c033459c6a19ca70a309f2c7c20102457" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p", "scenarioId": "tankcombat", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 44935 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "this-work-e/workspace/index.html", "bytes": 7301, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": null, "runtimeFailed": null } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 7301 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.007126388888888889 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 44935 }, { "source": "facts", "path": "outcome.durationMs", "value": 51310 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "schema-migration", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse", "runDir": "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse", "finishedAt": "2026-10-28T07:21:10.212Z", "rawSuccess": true, "rawReason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "durationMs": 162720, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 126.21, "nativePrefillTokensPerSec": 1942.95, "inputTokens": 113790, "outputTokens": 8002, "peakRssMb": 18461, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10603, "gpuMemoryTotalMb": 12227, "durationMs": 162720, "derived": { "durationMs": 152606, "meanTokensPerSec": 126.21, "genTokensPerSec": 126.21, "genTokens": 8002, "promptTokensPerSec": 1942.95, "peakRssMb": 18461, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 77606, "output": 3060 } } }, "usage": { "available": true, "totalInputTokens": 113790, "totalOutputTokens": 8002, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 77606, "todayTokensCached": 29829, "todayTokensOut": 3060, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 77606, "totalTokensCached": 29829, "totalTokensOut": 3060, "totalCost": 0, "medianOutputTokensPerSec": 137.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 137.7, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14440, "cachedInputTokens": 13814, "outputTokens": 76, "cost": 0, "durationMs": 13688, "outputTokensPerSec": 133.61005460197563, "at": "2026-10-28T07:21:00.435Z" }, "lastUpdated": "2026-10-28T07:21:00.435Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:21:03.124Z" } }, "source": "engine-log" }, "process": { "sampleCount": 28, "peakRssMb": 18461, "peakCpuPercent": 267.7, "firstAtMs": 1, "lastAtMs": 151186 }, "gpu": { "available": true, "sampleCount": 28, "peakUtilPercent": 99, "peakMemUsedMb": 10603, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 28624, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 149504, "totalTokens": 149504, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:18:27.492Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:28:50.924Z", "suiteId": "core", "scenarioId": "schema-migration", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Reviewed scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic code/test gates are scored separately.", "scenarioSourceSha256": "3d8dd177f12abdf9b935d258b56df3b06777af02face428721d4eb0eede0dc5c", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 28624, "totalMb": 31865 }, "evidenceHashes": { "result.json": "ceb6c1463b9903012990b21632bc4c6e647d418db0b773f5b9e27b27db4b0769", "score.json": "ce8c42b39fd7646779316384bb84564357fa967c5b53610bf323f5a4f8a6291e", "metrics.json": "f4a60d6b8a953e01f6695d35df5f5a21fcd9a10fd66b3fceac42cbb66e6bf601", "codex-inline-judge.json": "d8998baa5cf809e6b285c130e9aa69996723b38db39dc23b4ff675ab351b0e1f" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse", "scenarioId": "schema-migration", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24527 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "schema-migration", "bytes": 5961, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07748571428571428 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 16 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24527 }, { "source": "facts", "path": "outcome.durationMs", "value": 162720 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "schema-migration", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7", "runDir": "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7", "finishedAt": "2026-10-28T07:23:04.931Z", "rawSuccess": true, "rawReason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "durationMs": 114239, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 123.62, "nativePrefillTokensPerSec": 1936.26, "inputTokens": 73591, "outputTokens": 3815, "peakRssMb": 17586, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10603, "gpuMemoryTotalMb": 12227, "durationMs": 114239, "derived": { "durationMs": 107484, "meanTokensPerSec": 123.62, "genTokensPerSec": 123.62, "genTokens": 3815, "promptTokensPerSec": 1936.26, "peakRssMb": 17586, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 72799, "output": 2277 } } }, "usage": { "available": true, "totalInputTokens": 73591, "totalOutputTokens": 3815, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 72799, "todayTokensCached": 20649, "todayTokensOut": 2277, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 72799, "totalTokensCached": 20649, "totalTokensOut": 2277, "totalCost": 0, "medianOutputTokensPerSec": 141.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 141.4, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12988, "outputTokens": 767, "cost": 0, "durationMs": 13999, "outputTokensPerSec": 112.5288704128693, "at": "2026-10-28T07:22:58.099Z" }, "lastUpdated": "2026-10-28T07:22:58.099Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:23:01.695Z" } }, "source": "engine-log" }, "process": { "sampleCount": 20, "peakRssMb": 17586, "peakCpuPercent": 275.8, "firstAtMs": 1, "lastAtMs": 106109 }, "gpu": { "available": true, "sampleCount": 20, "peakUtilPercent": 99, "peakMemUsedMb": 10603, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 20, "peakUsedMb": 27743, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 149504, "totalTokens": 149504, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:21:10.692Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:28:50.932Z", "suiteId": "core", "scenarioId": "schema-migration", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Reviewed scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic code/test gates are scored separately.", "scenarioSourceSha256": "3d8dd177f12abdf9b935d258b56df3b06777af02face428721d4eb0eede0dc5c", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 20, "peakUsedMb": 27743, "totalMb": 31865 }, "evidenceHashes": { "result.json": "6d6b28167a7af4d9e198af0c09b15e4f9d5ad26652d281aa4409aa92ad5654df", "score.json": "a32329607047df11b1eeaf70dccb11a20c583f9c62f9cbfd61a126f48a81828d", "metrics.json": "f91f64c5315022c10eb33e3fe8bf88269c1f9de8302f0e877016f46a1a1afa73", "codex-inline-judge.json": "9899e80f631a6590cf60f423fa971ecd9c6d20616a11e41ce862126a88efd698" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7", "scenarioId": "schema-migration", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25840 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "schema-migration", "bytes": 6460, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.05439952380952381 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 12 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25840 }, { "source": "facts", "path": "outcome.durationMs", "value": 114239 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "schema-migration", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228", "runDir": "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228", "finishedAt": "2026-10-28T07:26:35.778Z", "rawSuccess": true, "rawReason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "durationMs": 209593, "rawFixedComposite": 9, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 118.49, "nativePrefillTokensPerSec": 1880.13, "inputTokens": 151266, "outputTokens": 8922, "peakRssMb": 21156, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10603, "gpuMemoryTotalMb": 12227, "durationMs": 209593, "derived": { "durationMs": 203020, "meanTokensPerSec": 118.49, "genTokensPerSec": 118.49, "genTokens": 8922, "promptTokensPerSec": 1880.13, "peakRssMb": 21156, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 179298, "output": 5477 } } }, "usage": { "available": true, "totalInputTokens": 151266, "totalOutputTokens": 8922, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 179298, "todayTokensCached": 101968, "todayTokensOut": 5477, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 179298, "totalTokensCached": 101968, "totalTokensOut": 5477, "totalCost": 0, "medianOutputTokensPerSec": 137.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 137.1, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19371, "outputTokens": 50, "cost": 0, "durationMs": 22999, "outputTokensPerSec": 150.79428581979775, "at": "2026-10-28T07:26:26.183Z" }, "lastUpdated": "2026-10-28T07:26:26.183Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:26:32.203Z" } }, "source": "engine-log" }, "process": { "sampleCount": 37, "peakRssMb": 21156, "peakCpuPercent": 261.5, "firstAtMs": 1, "lastAtMs": 201437 }, "gpu": { "available": true, "sampleCount": 37, "peakUtilPercent": 100, "peakMemUsedMb": 10603, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 37, "peakUsedMb": 30985, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 149504, "totalTokens": 149504, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:23:06.185Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:28:50.940Z", "suiteId": "core", "scenarioId": "schema-migration", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Reviewed scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic code/test gates are scored separately.", "scenarioSourceSha256": "3d8dd177f12abdf9b935d258b56df3b06777af02face428721d4eb0eede0dc5c", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 37, "peakUsedMb": 30985, "totalMb": 31865 }, "evidenceHashes": { "result.json": "beff2264652f99ecb28601de1b24a0be4ea28b8dfe8a264c957e951319af8261", "score.json": "520f31e31a9e150f8b43e82fe4bc9ad4bb4c2454e006dde6a0812080fe33c7cd", "metrics.json": "32ed08caff1b858eaba226689f76b0fcad5d54e4853f23e5219255307917d61e", "codex-inline-judge.json": "d975af701eec2c8e5706455b2430d1d3f5a82aa1e017a74275a704bceb781897" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228", "scenarioId": "schema-migration", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24659 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "schema-migration", "bytes": 5014, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.09980619047619048 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 20 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24659 }, { "source": "facts", "path": "outcome.durationMs", "value": 209593 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "failing-tests-spec", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn", "runDir": "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn", "finishedAt": "2026-10-28T07:27:10.796Z", "rawSuccess": true, "rawReason": "vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "durationMs": 31136, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 136.21, "nativePrefillTokensPerSec": 2022.87, "inputTokens": 13671, "outputTokens": 792, "peakRssMb": 11516, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 31136, "derived": { "durationMs": 25402, "meanTokensPerSec": 136.21, "genTokensPerSec": 136.21, "genTokens": 792, "promptTokensPerSec": 2022.87, "peakRssMb": 11516, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6624, "output": 654 } } }, "usage": { "available": true, "totalInputTokens": 13671, "totalOutputTokens": 792, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6624, "todayTokensCached": 0, "todayTokensOut": 654, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6624, "totalTokensCached": 0, "totalTokensOut": 654, "totalCost": 0, "medianOutputTokensPerSec": 136.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 136.1, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6624, "outputTokens": 654, "cost": 0, "durationMs": 20182, "outputTokensPerSec": 136.11670099809186, "at": "2026-10-28T07:27:05.229Z" }, "lastUpdated": "2026-10-28T07:27:05.229Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:27:08.019Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 11516, "peakCpuPercent": 250, "firstAtMs": 1, "lastAtMs": 22285 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21208, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:26:39.660Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:28:50.892Z", "suiteId": "core", "scenarioId": "failing-tests-spec", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Reviewed scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic code/test gates are scored separately.", "scenarioSourceSha256": "6b8ae9790b33d38869d7f0dfbd85cd6c66646012d63d79a6a71d51de1a2f2604", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21208, "totalMb": 31865 }, "evidenceHashes": { "result.json": "02bd31d9777ef8ab919142a694557458764a2d3f5fa0dc09d867c198625f13f7", "score.json": "dbc291b95f564f4cd793c3c610e4794af0e84e8eea54a132b8c9827862a5c9b9", "metrics.json": "18926c0e56292b3bc9f0a26731627c9d2e5762b51a7b6aed15aae3fac8bfaefe", "codex-inline-judge.json": "eaa0d5b7dace31b420761e8209e67e6afb17b7e1a20aa8f2cffb8dd4937b4c3c" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn", "scenarioId": "failing-tests-spec", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25560 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "failing-tests-spec", "bytes": 2469, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.020757333333333332 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25560 }, { "source": "facts", "path": "outcome.durationMs", "value": 31136 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "failing-tests-spec", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t", "runDir": "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t", "finishedAt": "2026-10-28T07:27:41.712Z", "rawSuccess": true, "rawReason": "vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "durationMs": 29954, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 139.69, "nativePrefillTokensPerSec": 2016.93, "inputTokens": 13671, "outputTokens": 776, "peakRssMb": 11511, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 29954, "derived": { "durationMs": 24146, "meanTokensPerSec": 139.69, "genTokensPerSec": 139.69, "genTokens": 776, "promptTokensPerSec": 2016.93, "peakRssMb": 11511, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6624, "output": 638 } } }, "usage": { "available": true, "totalInputTokens": 13671, "totalOutputTokens": 776, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6624, "todayTokensCached": 0, "todayTokensOut": 638, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6624, "totalTokensCached": 0, "totalTokensOut": 638, "totalCost": 0, "medianOutputTokensPerSec": 140.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 140.2, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6624, "outputTokens": 638, "cost": 0, "durationMs": 19964, "outputTokensPerSec": 140.16832521515948, "at": "2026-10-28T07:27:37.348Z" }, "lastUpdated": "2026-10-28T07:27:37.348Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:27:38.869Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 11511, "peakCpuPercent": 244.8, "firstAtMs": 1, "lastAtMs": 22297 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21235, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:27:11.758Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:28:50.904Z", "suiteId": "core", "scenarioId": "failing-tests-spec", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Reviewed scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic code/test gates are scored separately.", "scenarioSourceSha256": "6b8ae9790b33d38869d7f0dfbd85cd6c66646012d63d79a6a71d51de1a2f2604", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21235, "totalMb": 31865 }, "evidenceHashes": { "result.json": "dfdd2315f5b909533fccfe132e8075d66a9b3841ea932d3f1e4030cdb0ce8774", "score.json": "2b6c7ca1cfe14234fdb17871e30eba6458267372b51f777e1fb4a542d8d67f8e", "metrics.json": "da8a1e7d2804482c3a7851d979d31549af3de65c065d9fd5319c59f4b12d69ed", "codex-inline-judge.json": "e138a0809913ccdbd5fdfad0154a95d87e3c4441fd17591f7c70c8a0168405b0" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t", "scenarioId": "failing-tests-spec", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25585 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "failing-tests-spec", "bytes": 2231, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.019969333333333332 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25585 }, { "source": "facts", "path": "outcome.durationMs", "value": 29954 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "failing-tests-spec", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh", "runDir": "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh", "finishedAt": "2026-10-28T07:29:59.635Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on src/machine.ts; missing signals stayed unchanged: tsc-clean. Last failure: tsc --noEmit failed (exit 2): src/machine.ts(31,7): error TS2322: Type 'string | undefined' is not assignable to type 'string'.", "durationMs": 136793, "rawFixedComposite": 6.1, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 148.97, "nativePrefillTokensPerSec": 1928.64, "inputTokens": 135546, "outputTokens": 1427, "peakRssMb": 16882, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 136793, "derived": { "durationMs": 130446, "meanTokensPerSec": 148.97, "genTokensPerSec": 148.97, "genTokens": 1427, "promptTokensPerSec": 1928.64, "peakRssMb": 16882, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 81021, "output": 1206 } } }, "usage": { "available": true, "totalInputTokens": 135546, "totalOutputTokens": 1427, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 81021, "todayTokensCached": 24994, "todayTokensOut": 1206, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 81021, "totalTokensCached": 24994, "totalTokensOut": 1206, "totalCost": 0, "medianOutputTokensPerSec": 156, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 156, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8699, "outputTokens": 474, "cost": 0, "durationMs": 15567, "outputTokensPerSec": 153.0478855447728, "at": "2026-10-28T07:29:50.194Z" }, "lastUpdated": "2026-10-28T07:29:50.194Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:29:56.267Z" } }, "source": "engine-log" }, "process": { "sampleCount": 24, "peakRssMb": 16882, "peakCpuPercent": 231.7, "firstAtMs": 0, "lastAtMs": 128567 }, "gpu": { "available": true, "sampleCount": 24, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 24, "peakUsedMb": 26666, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:27:42.842Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:33:05.589Z", "suiteId": "core", "scenarioId": "failing-tests-spec", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 24, "peakUsedMb": 26666, "totalMb": 31865 }, "evidenceHashes": { "result.json": "21f0495aa17dfbc7201c4e6eee77b533023a517b35260c06fde252b1e421b564", "score.json": "61c6dde3ae93bbc263209e506f484b5b9f1f7a91649405f6c25eb587e989e884", "metrics.json": "db216c644ea750aaeecac7e479f4dcb205904f74d76d9d83c6260b90835321d7", "codex-inline-judge.json": "bd94cb2c0060b6155c44832b4012f0bdd139e636b0ae6506bf341a1782e09c79" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh", "scenarioId": "failing-tests-spec", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on src/machine.ts; missing signals stayed unchanged: tsc-clean. Last failure: tsc --noEmit failed (exit 2): src/machine.ts(31,7): error TS2322: Type 'string | undefined' is not assignable to type 'string'." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23399 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "failing-tests-spec", "bytes": 1761, "score": 3, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green" ], "failReason": "tsc --noEmit failed (exit 2): src/machine.ts(31,7): error TS2322: Type 'string | undefined' is not assignable to type 'string'." } } ] }, "quality": { "score": 7.5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 75% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23399 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "failing-tests-spec", "bytes": 1761, "score": 3, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green" ], "failReason": "tsc --noEmit failed (exit 2): src/machine.ts(31,7): error TS2322: Type 'string | undefined' is not assignable to type 'string'." } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.09119533333333334 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 16 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23399 }, { "source": "facts", "path": "outcome.durationMs", "value": 136793 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6.1, "band": "needs-tuning" } }, { "suiteId": "core", "scenarioId": "symptom-debug", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu", "runDir": "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu", "finishedAt": "2026-10-28T07:30:46.665Z", "rawSuccess": true, "rawReason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "durationMs": 43281, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 117.52, "nativePrefillTokensPerSec": 1786.98, "inputTokens": 5570, "outputTokens": 2471, "peakRssMb": 11406, "gpuAvailable": true, "peakGpuUtilPercent": 84, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 43281, "derived": { "durationMs": 34798, "meanTokensPerSec": 117.52, "genTokensPerSec": 117.52, "genTokens": 2471, "promptTokensPerSec": 1786.98, "peakRssMb": 11406, "peakGpuUtilPercent": 84, "billing": null }, "usage": { "available": true, "totalInputTokens": 5570, "totalOutputTokens": 2471, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:30:41.113Z" } }, "source": "engine-log" }, "process": { "sampleCount": 7, "peakRssMb": 11406, "peakCpuPercent": 258.7, "firstAtMs": 0, "lastAtMs": 33647 }, "gpu": { "available": true, "sampleCount": 7, "peakUtilPercent": 84, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 21465, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:30:03.384Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:33:05.661Z", "suiteId": "core", "scenarioId": "symptom-debug", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 21465, "totalMb": 31865 }, "evidenceHashes": { "result.json": "2399e4f4d66d4f8c86a9b8bef26f89602091ec6f3717bff78ecce925c43dc9b0", "score.json": "25d16140338ef308fbc83b31d1ef892ccc23a0eaf0f769a3b748b5bca51ab6ac", "metrics.json": "0cb991c8be5a5d0127632695171cd5275f1f0cb34b8fbf862f332610ee391649", "codex-inline-judge.json": "7a8e48d12af8a637dc8a8c0d0f51f05dd93b52f65d65850d524b3d16ed20e53c" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu", "scenarioId": "symptom-debug", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 37086 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "symptom-debug", "bytes": 1189, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.0360675 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 37086 }, { "source": "facts", "path": "outcome.durationMs", "value": 43281 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "symptom-debug", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283", "runDir": "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283", "finishedAt": "2026-10-28T07:31:38.959Z", "rawSuccess": true, "rawReason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "durationMs": 50825, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 129.73, "nativePrefillTokensPerSec": 1646.2, "inputTokens": 7034, "outputTokens": 2630, "peakRssMb": 13980, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 50825, "derived": { "durationMs": 40319, "meanTokensPerSec": 129.73, "genTokensPerSec": 129.73, "genTokens": 2630, "promptTokensPerSec": 1646.2, "peakRssMb": 13980, "peakGpuUtilPercent": 99, "billing": null }, "usage": { "available": true, "totalInputTokens": 7034, "totalOutputTokens": 2630, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:31:31.504Z" } }, "source": "engine-log" }, "process": { "sampleCount": 8, "peakRssMb": 13980, "peakCpuPercent": 261.7, "firstAtMs": 2, "lastAtMs": 39192 }, "gpu": { "available": true, "sampleCount": 8, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 24042, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:30:48.134Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:33:05.668Z", "suiteId": "core", "scenarioId": "symptom-debug", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 24042, "totalMb": 31865 }, "evidenceHashes": { "result.json": "925c84c3cad58533d0481db094723ecf4289229859349a7f668ce72821a0f97f", "score.json": "edd35cfc8bc9201c51611612d0d8c6d6b1eae7e8ad91ad4c89d4ec169d453486", "metrics.json": "2bb71e812327b11a00b7f16b72b323277b2060b04d8e86ca867badc9c807e646", "codex-inline-judge.json": "3921223ae24605d1d451baf429afac4d1058b44bc513cdab0ce960b4f72963d6" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283", "scenarioId": "symptom-debug", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 37805 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "symptom-debug", "bytes": 1189, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.042354166666666665 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 7 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 37805 }, { "source": "facts", "path": "outcome.durationMs", "value": 50825 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "symptom-debug", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj", "runDir": "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj", "finishedAt": "2026-10-28T07:32:14.907Z", "rawSuccess": true, "rawReason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "durationMs": 35492, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 118.38, "nativePrefillTokensPerSec": 1814.82, "inputTokens": 14149, "outputTokens": 737, "peakRssMb": 14531, "gpuAvailable": true, "peakGpuUtilPercent": 90, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 35492, "derived": { "durationMs": 29210, "meanTokensPerSec": 118.38, "genTokensPerSec": 118.38, "genTokens": 737, "promptTokensPerSec": 1814.82, "peakRssMb": 14531, "peakGpuUtilPercent": 90, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7125, "output": 110 } } }, "usage": { "available": true, "totalInputTokens": 14149, "totalOutputTokens": 737, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7125, "todayTokensCached": 0, "todayTokensOut": 110, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7125, "totalTokensCached": 0, "totalTokensOut": 110, "totalCost": 0, "medianOutputTokensPerSec": 140.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 140.6, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7125, "outputTokens": 110, "cost": 0, "durationMs": 5269, "outputTokensPerSec": 140.58620578467094, "at": "2026-10-28T07:32:08.863Z" }, "lastUpdated": "2026-10-28T07:32:08.863Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:32:11.610Z" } }, "source": "engine-log" }, "process": { "sampleCount": 6, "peakRssMb": 14531, "peakCpuPercent": 248.1, "firstAtMs": 1, "lastAtMs": 28132 }, "gpu": { "available": true, "sampleCount": 6, "peakUtilPercent": 90, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 24570, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:31:39.415Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:33:05.674Z", "suiteId": "core", "scenarioId": "symptom-debug", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 24570, "totalMb": 31865 }, "evidenceHashes": { "result.json": "46787f496c25b924ecb125ee09a85b2b0f262440b832c62204ae66732da17cf0", "score.json": "9c0d42f251eae98b5da4e0090d9dc7064be988d547cc64d9941f3c3f42297791", "metrics.json": "b6ac11bd2c23ef5a832d105d5d3769f71f55e63e93de09bc416bccb1728c5c75", "codex-inline-judge.json": "8b182fa2a92113d396c5f3f272d4346c26dbaaff3261233492b4070ffc610d0f" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj", "scenarioId": "symptom-debug", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 29425 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "symptom-debug", "bytes": 1189, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.029576666666666668 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 7 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 29425 }, { "source": "facts", "path": "outcome.durationMs", "value": 35492 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "data-wrangle", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk", "runDir": "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk", "finishedAt": "2026-10-28T07:36:44.319Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/customers.json; missing signals stayed unchanged: row-count. Last failure: expected exactly 56 records after dedup, got 51", "durationMs": 265818, "rawFixedComposite": 5.6, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 147.47, "nativePrefillTokensPerSec": 1825.18, "inputTokens": 73954, "outputTokens": 19846, "peakRssMb": 17193, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 265818, "derived": { "durationMs": 258976, "meanTokensPerSec": 147.47, "genTokensPerSec": 147.47, "genTokens": 19846, "promptTokensPerSec": 1825.18, "peakRssMb": 17193, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 78199, "output": 8472 } } }, "usage": { "available": true, "totalInputTokens": 73954, "totalOutputTokens": 19846, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 78199, "todayTokensCached": 35617, "todayTokensOut": 8472, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 78199, "totalTokensCached": 35617, "totalTokensOut": 8472, "totalCost": 0, "medianOutputTokensPerSec": 153.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 153.4, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19420, "outputTokens": 3150, "cost": 0, "durationMs": 94613, "outputTokensPerSec": 153.65293632029957, "at": "2026-10-28T07:36:34.665Z" }, "lastUpdated": "2026-10-28T07:36:34.665Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:36:40.474Z" } }, "source": "engine-log" }, "process": { "sampleCount": 47, "peakRssMb": 17193, "peakCpuPercent": 355.7, "firstAtMs": 0, "lastAtMs": 257894 }, "gpu": { "available": true, "sampleCount": 47, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 47, "peakUsedMb": 27744, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:32:18.501Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:40:35.866Z", "suiteId": "core", "scenarioId": "data-wrangle", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 47, "peakUsedMb": 27744, "totalMb": 31865 }, "evidenceHashes": { "result.json": "e5c47f97a5beefa488f9682d395bb628273e5c707cabb4324ce69d78ddf30ce0", "score.json": "368c155fa84c88c64b976770fc5a45de9650c887a34754403c0201eacb470e87", "metrics.json": "e27ac9f67d42105c4a78ca679fb4bc80d5fb624cbd326d05a4cec2e1399d5ea4", "codex-inline-judge.json": "b5883d11eac01f96154422aa176ed789d490230935f40f476049600160dedeb9" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk", "scenarioId": "data-wrangle", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/customers.json; missing signals stayed unchanged: row-count. Last failure: expected exactly 56 records after dedup, got 51" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 30886 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 6354, "score": 6, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved" ], "failReason": "expected exactly 56 records after dedup, got 51" } } ] }, "quality": { "score": 5.5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 54.5% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 30886 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 6354, "score": 6, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved" ], "failReason": "expected exactly 56 records after dedup, got 51" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.09845111111111111 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 15 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 30886 }, { "source": "facts", "path": "outcome.durationMs", "value": 265818 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.6, "band": "needs-tuning" } }, { "suiteId": "core", "scenarioId": "data-wrangle", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k", "runDir": "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k", "finishedAt": "2026-10-28T07:48:51.714Z", "rawSuccess": false, "rawReason": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 4 while the failing detail kept changing; latest checked file out/customers.json. Last failure: email \"wanda.m@example.com\" appears more than once — dedup is missing", "durationMs": 726372, "rawFixedComposite": 3.4, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 135.74, "nativePrefillTokensPerSec": 1868.65, "inputTokens": 181883, "outputTokens": 73372, "peakRssMb": 18140, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 726372, "derived": { "durationMs": 718290, "meanTokensPerSec": 135.74, "genTokensPerSec": 135.74, "genTokens": 73372, "promptTokensPerSec": 1868.65, "peakRssMb": 18140, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 136470, "output": 29648 } } }, "usage": { "available": true, "totalInputTokens": 181883, "totalOutputTokens": 73372, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 136470, "todayTokensCached": 72380, "todayTokensOut": 29648, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 136470, "totalTokensCached": 72380, "totalTokensOut": 29648, "totalCost": 0, "medianOutputTokensPerSec": 132.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 132.6, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10587, "cachedInputTokens": 9547, "outputTokens": 1283, "cost": 0, "durationMs": 87728, "outputTokensPerSec": 124.98624127667692, "at": "2026-10-28T07:48:44.915Z" }, "lastUpdated": "2026-10-28T07:48:44.915Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:48:46.875Z" } }, "source": "engine-log" }, "process": { "sampleCount": 129, "peakRssMb": 18140, "peakCpuPercent": 360.3, "firstAtMs": 1, "lastAtMs": 717251 }, "gpu": { "available": true, "sampleCount": 129, "peakUtilPercent": 100, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 129, "peakUsedMb": 28399, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:36:45.342Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:49:40.272Z", "suiteId": "core", "scenarioId": "data-wrangle", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 129, "peakUsedMb": 28399, "totalMb": 31865 }, "evidenceHashes": { "result.json": "b0463dab6d7b99146c6164fccd555c59847f8dac242e216411b854c4cf6bf63c", "score.json": "b0753249489021cec0287c96e3ebbb3280e2d80f824ff4e66b948366d1e94225", "metrics.json": "1ab424f667fb7ab4d557ded7eb71e3467b33c55518a00db0a21bbaaed0e59724", "codex-inline-judge.json": "7df76c89ff6d67683413adadac02f677c3156646922ff49e4d67de62d8cd44a1" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k", "scenarioId": "data-wrangle", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 4 while the failing detail kept changing; latest checked file out/customers.json. Last failure: email \"wanda.m@example.com\" appears more than once — dedup is missing" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 32042 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 5516, "score": 4, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso" ], "failReason": "email \"wanda.m@example.com\" appears more than once — dedup is missing" } } ] }, "quality": { "score": 3.6, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 36.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 32042 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 5516, "score": 4, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso" ], "failReason": "email \"wanda.m@example.com\" appears more than once — dedup is missing" } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2690266666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 25 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 32042 }, { "source": "facts", "path": "outcome.durationMs", "value": 726372 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.4, "band": "framework-gap" } }, { "suiteId": "core", "scenarioId": "data-wrangle", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l", "runDir": "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l", "finishedAt": "2026-10-28T07:58:08.020Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/customers.json; missing signals stayed unchanged: unique-emails. Last failure: email \"alice.archer@example.com\" appears more than once — dedup is missing", "durationMs": 555321, "rawFixedComposite": 3.4, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 141.01, "nativePrefillTokensPerSec": 1832.31, "inputTokens": 93616, "outputTokens": 27826, "peakRssMb": 18188, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 555321, "derived": { "durationMs": 548783, "meanTokensPerSec": 141.01, "genTokensPerSec": 141.01, "genTokens": 27826, "promptTokensPerSec": 1832.31, "peakRssMb": 18188, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 132357, "output": 13052 } } }, "usage": { "available": true, "totalInputTokens": 93616, "totalOutputTokens": 27826, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 132357, "todayTokensCached": 104889, "todayTokensOut": 13052, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 132357, "totalTokensCached": 104889, "totalTokensOut": 13052, "totalCost": 0, "medianOutputTokensPerSec": 134.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 134.9, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14975, "outputTokens": 3707, "cost": 0, "durationMs": 32688, "outputTokensPerSec": 154.25708770943126, "at": "2026-10-28T07:57:59.056Z" }, "lastUpdated": "2026-10-28T07:57:59.056Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:58:04.863Z" } }, "source": "engine-log" }, "process": { "sampleCount": 99, "peakRssMb": 18188, "peakCpuPercent": 270.7, "firstAtMs": 0, "lastAtMs": 547717 }, "gpu": { "available": true, "sampleCount": 99, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 99, "peakUsedMb": 29483, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:48:52.699Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T07:58:52.911Z", "suiteId": "core", "scenarioId": "data-wrangle", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 99, "peakUsedMb": 29483, "totalMb": 31865 }, "evidenceHashes": { "result.json": "18a23fcd6929ee370f3f488ac2c1c6095cae1c7f2d17c666bc4408e07dc119ff", "score.json": "1e2ea00aae8d907e058c93ec43f1a448c87571dbf096aa065604a017bf2147ee", "metrics.json": "0c3e17cb52f6d17939f9f44e5c995443bfd1aff7823d6ae5a38af73f6589f0c2", "codex-inline-judge.json": "b028864400d0dcf6d961b6e285980518ac99f6dcfacea6776424f7122dff9c27" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l", "scenarioId": "data-wrangle", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/customers.json; missing signals stayed unchanged: unique-emails. Last failure: email \"alice.archer@example.com\" appears more than once — dedup is missing" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34421 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 7489, "score": 4, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso" ], "failReason": "email \"alice.archer@example.com\" appears more than once — dedup is missing" } } ] }, "quality": { "score": 3.6, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 36.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34421 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 7489, "score": 4, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso" ], "failReason": "email \"alice.archer@example.com\" appears more than once — dedup is missing" } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.20567444444444444 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 23 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34421 }, { "source": "facts", "path": "outcome.durationMs", "value": 555321 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.4, "band": "framework-gap" } }, { "suiteId": "core", "scenarioId": "incident-postmortem", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4", "runDir": "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4", "finishedAt": "2026-10-28T07:59:49.184Z", "rawSuccess": true, "rawReason": "all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "durationMs": 97451, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 107.52, "nativePrefillTokensPerSec": 1940, "inputTokens": 38515, "outputTokens": 5929, "peakRssMb": 14794, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 97451, "derived": { "durationMs": 90618, "meanTokensPerSec": 107.52, "genTokensPerSec": 107.52, "genTokens": 5929, "promptTokensPerSec": 1940, "peakRssMb": 14794, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 27026, "output": 3325 } } }, "usage": { "available": true, "totalInputTokens": 38515, "totalOutputTokens": 5929, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 27026, "todayTokensCached": 16984, "todayTokensOut": 3325, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 27026, "totalTokensCached": 16984, "totalTokensOut": 3325, "totalCost": 0, "medianOutputTokensPerSec": 105.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 105.4, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8835, "outputTokens": 3199, "cost": 0, "durationMs": 36408, "outputTokensPerSec": 102.28302557539399, "at": "2026-10-28T07:59:43.550Z" }, "lastUpdated": "2026-10-28T07:59:43.550Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T07:59:45.743Z" } }, "source": "engine-log" }, "process": { "sampleCount": 17, "peakRssMb": 14794, "peakCpuPercent": 266.9, "firstAtMs": 0, "lastAtMs": 89544 }, "gpu": { "available": true, "sampleCount": 17, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 25905, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:58:11.733Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.443Z", "suiteId": "core", "scenarioId": "incident-postmortem", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 25905, "totalMb": 31865 }, "evidenceHashes": { "result.json": "9e6b81c23ff2cd12472a2fb4bb4e6c0a73c01930fc23cff86f8bcc0a2c8e263e", "score.json": "ae4109c89d0c1b2af00c69e738dcfe23dffde684a2835ec18f9da46e4ae4ccca", "metrics.json": "e17d1b24c1f0eebf3ec58be6776dcfcfbfa4644084ce0d0b44b7ff626dcb9a54", "codex-inline-judge.json": "cacd6cdb2d3f482a3486a998f1cc1d2c166cef52265d53193ff8af6fcd350116" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4", "scenarioId": "incident-postmortem", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51469 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "incident-postmortem", "bytes": 8859, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.04060458333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 5 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51469 }, { "source": "facts", "path": "outcome.durationMs", "value": 97451 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "incident-postmortem", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6", "runDir": "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6", "finishedAt": "2026-10-28T08:01:42.944Z", "rawSuccess": true, "rawReason": "all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "durationMs": 113453, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 106.14, "nativePrefillTokensPerSec": 1927.35, "inputTokens": 40396, "outputTokens": 6656, "peakRssMb": 15454, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 113453, "derived": { "durationMs": 107195, "meanTokensPerSec": 106.14, "genTokensPerSec": 106.14, "genTokens": 6656, "promptTokensPerSec": 1927.35, "peakRssMb": 15454, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28366, "output": 3009 } } }, "usage": { "available": true, "totalInputTokens": 40396, "totalOutputTokens": 6656, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 28366, "todayTokensCached": 17576, "todayTokensOut": 3009, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 28366, "totalTokensCached": 17576, "totalTokensOut": 3009, "totalCost": 0, "medianOutputTokensPerSec": 100, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 100, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9559, "outputTokens": 2737, "cost": 0, "durationMs": 33223, "outputTokensPerSec": 99.00190090163015, "at": "2026-10-28T08:01:34.404Z" }, "lastUpdated": "2026-10-28T08:01:34.404Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:01:39.668Z" } }, "source": "engine-log" }, "process": { "sampleCount": 20, "peakRssMb": 15454, "peakCpuPercent": 264.1, "firstAtMs": 1, "lastAtMs": 106134 }, "gpu": { "available": true, "sampleCount": 20, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 20, "peakUsedMb": 26600, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T07:59:49.491Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.450Z", "suiteId": "core", "scenarioId": "incident-postmortem", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 20, "peakUsedMb": 26600, "totalMb": 31865 }, "evidenceHashes": { "result.json": "1d386036dcf8c4f651230920d1eca2b1f3a58af596a99ef03c63bcd60de651cf", "score.json": "a8cd58c8993a9d07ad0c988227bc8135d061bcb29732176337996e58d45e361c", "metrics.json": "4a2dac867512391ad0b76a3064fb4be513a670a8ed71811d50390b36ad305624", "codex-inline-judge.json": "c6db839066e370d3d86507a508580c6876ebdd11472ff3350e648886f4d1ca35" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6", "scenarioId": "incident-postmortem", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 61725 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "incident-postmortem", "bytes": 6791, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.04727208333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 6 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 61725 }, { "source": "facts", "path": "outcome.durationMs", "value": 113453 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "incident-postmortem", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4", "runDir": "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4", "finishedAt": "2026-10-28T08:03:43.188Z", "rawSuccess": true, "rawReason": "all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "durationMs": 119737, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 101.82, "nativePrefillTokensPerSec": 1915, "inputTokens": 41933, "outputTokens": 6556, "peakRssMb": 15785, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 119737, "derived": { "durationMs": 112585, "meanTokensPerSec": 101.82, "genTokensPerSec": 101.82, "genTokens": 6556, "promptTokensPerSec": 1915, "peakRssMb": 15785, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 38811, "output": 3244 } } }, "usage": { "available": true, "totalInputTokens": 41933, "totalOutputTokens": 6556, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 38811, "todayTokensCached": 27119, "todayTokensOut": 3244, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 38811, "totalTokensCached": 27119, "totalTokensOut": 3244, "totalCost": 0, "medianOutputTokensPerSec": 86.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 86.9, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10386, "cachedInputTokens": 9096, "outputTokens": 1557, "cost": 0, "durationMs": 19186, "outputTokensPerSec": 86.86852694091142, "at": "2026-10-28T08:03:35.452Z" }, "lastUpdated": "2026-10-28T08:03:35.452Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:03:38.988Z" } }, "source": "engine-log" }, "process": { "sampleCount": 21, "peakRssMb": 15785, "peakCpuPercent": 263.3, "firstAtMs": 1, "lastAtMs": 111529 }, "gpu": { "available": true, "sampleCount": 21, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 21, "peakUsedMb": 26840, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:01:43.451Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.458Z", "suiteId": "core", "scenarioId": "incident-postmortem", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 21, "peakUsedMb": 26840, "totalMb": 31865 }, "evidenceHashes": { "result.json": "d809c134f6251e48e29101604c5bad2fefefff2ba7b9d3a817d8508bffc258f1", "score.json": "95e0d3fdb73baa320d4f2221faa3b44608732cd5a35e4994d3f9f0f804504f15", "metrics.json": "971a7f7eff40ae42f717c024b3c516122890d994a2c645cf5040805c0cf78a9e", "codex-inline-judge.json": "0f489216fac929c43e0f452fac286b4f1a01293c84f48d74480a554013b2a75f" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4", "scenarioId": "incident-postmortem", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 58102 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "incident-postmortem", "bytes": 8976, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.049890416666666666 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 6 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 58102 }, { "source": "facts", "path": "outcome.durationMs", "value": 119737 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0", "runDir": "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0", "finishedAt": "2026-10-28T08:05:01.329Z", "rawSuccess": true, "rawReason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "durationMs": 74342, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 122.07, "nativePrefillTokensPerSec": 1947.52, "inputTokens": 53591, "outputTokens": 2195, "peakRssMb": 15542, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 74342, "derived": { "durationMs": 67896, "meanTokensPerSec": 122.07, "genTokensPerSec": 122.07, "genTokens": 2195, "promptTokensPerSec": 1947.52, "peakRssMb": 15542, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 26691, "output": 1631 } } }, "usage": { "available": true, "totalInputTokens": 53591, "totalOutputTokens": 2195, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 26691, "todayTokensCached": 10669, "todayTokensOut": 1631, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 26691, "totalTokensCached": 10669, "totalTokensOut": 1631, "totalCost": 0, "medianOutputTokensPerSec": 120.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 120.2, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8145, "outputTokens": 129, "cost": 0, "durationMs": 14117, "outputTokensPerSec": 139.03734021568167, "at": "2026-10-28T08:04:54.829Z" }, "lastUpdated": "2026-10-28T08:04:54.829Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:04:57.895Z" } }, "source": "engine-log" }, "process": { "sampleCount": 13, "peakRssMb": 15542, "peakCpuPercent": 263.1, "firstAtMs": 3, "lastAtMs": 66834 }, "gpu": { "available": true, "sampleCount": 13, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 13, "peakUsedMb": 26748, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:03:46.987Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.467Z", "suiteId": "core", "scenarioId": "ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 13, "peakUsedMb": 26748, "totalMb": 31865 }, "evidenceHashes": { "result.json": "80a40bbd57e54ad2515c93ffaef1f33d8df2cc8d6e6e49e512f0567ddfd8f68c", "score.json": "ca64b34deea93758050eb00b8b9fce6fbc457b9067efbea5d91b627a02e81225", "metrics.json": "cd06f075b294a63caa3e264bcd106497b67b379fdbb6763ab0310f74631f6ee9", "codex-inline-judge.json": "3b6dc7dd1a3a1baff02311e840c6fe53639ff51e125a46b89ff0be0edd1cc205" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0", "scenarioId": "ops-runbook-anomaly", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 21276 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "ops-runbook-anomaly", "bytes": 476, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.03540095238095238 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 12 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 21276 }, { "source": "facts", "path": "outcome.durationMs", "value": 74342 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje", "runDir": "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje", "finishedAt": "2026-10-28T08:06:27.461Z", "rawSuccess": true, "rawReason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "durationMs": 85813, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 122.09, "nativePrefillTokensPerSec": 1944.05, "inputTokens": 73904, "outputTokens": 1243, "peakRssMb": 15890, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 85813, "derived": { "durationMs": 78866, "meanTokensPerSec": 122.09, "genTokensPerSec": 122.09, "genTokens": 1243, "promptTokensPerSec": 1944.05, "peakRssMb": 15890, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 36479, "output": 452 } } }, "usage": { "available": true, "totalInputTokens": 73904, "totalOutputTokens": 1243, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 36479, "todayTokensCached": 19682, "todayTokensOut": 452, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 36479, "totalTokensCached": 19682, "totalTokensOut": 452, "totalCost": 0, "medianOutputTokensPerSec": 130.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 130.4, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9556, "cachedInputTokens": 8334, "outputTokens": 122, "cost": 0, "durationMs": 12071, "outputTokensPerSec": 139.1325945124265, "at": "2026-10-28T08:06:18.638Z" }, "lastUpdated": "2026-10-28T08:06:18.638Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:06:23.648Z" } }, "source": "engine-log" }, "process": { "sampleCount": 15, "peakRssMb": 15890, "peakCpuPercent": 226.1, "firstAtMs": 0, "lastAtMs": 77808 }, "gpu": { "available": true, "sampleCount": 15, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 26955, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:05:01.648Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.475Z", "suiteId": "core", "scenarioId": "ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 26955, "totalMb": 31865 }, "evidenceHashes": { "result.json": "5a82fedc41e214db576fe887a0d350e5fba9ad480f1ac686a03090049a6ae3f5", "score.json": "386c61a81583c64999cf0f82d4867e312d044442f5df38750968285421327960", "metrics.json": "9e5db68a772b17a9f44122290ff7dfa64a6121aa684aa8714cdb4d276b2faa2a", "codex-inline-judge.json": "a5268a77d4aeb3efcc4ead7bb3d2ed3f0c2cd14bbebd53387b2ce970b301c726" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje", "scenarioId": "ops-runbook-anomaly", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 20256 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "ops-runbook-anomaly", "bytes": 400, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.040863333333333335 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 20256 }, { "source": "facts", "path": "outcome.durationMs", "value": 85813 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab", "runDir": "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab", "finishedAt": "2026-10-28T08:07:43.436Z", "rawSuccess": true, "rawReason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "durationMs": 74794, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 115.02, "nativePrefillTokensPerSec": 1963.53, "inputTokens": 53455, "outputTokens": 1802, "peakRssMb": 15537, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 74794, "derived": { "durationMs": 68632, "meanTokensPerSec": 115.02, "genTokensPerSec": 115.02, "genTokens": 1802, "promptTokensPerSec": 1963.53, "peakRssMb": 15537, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 26645, "output": 1311 } } }, "usage": { "available": true, "totalInputTokens": 53455, "totalOutputTokens": 1802, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 26645, "todayTokensCached": 10668, "todayTokensOut": 1311, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 26645, "totalTokensCached": 10668, "totalTokensOut": 1311, "totalCost": 0, "medianOutputTokensPerSec": 111.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 111.8, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8123, "outputTokens": 141, "cost": 0, "durationMs": 14045, "outputTokensPerSec": 129.50647856159006, "at": "2026-10-28T08:07:37.310Z" }, "lastUpdated": "2026-10-28T08:07:37.310Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:07:40.318Z" } }, "source": "engine-log" }, "process": { "sampleCount": 13, "peakRssMb": 15537, "peakCpuPercent": 262.8, "firstAtMs": 0, "lastAtMs": 67559 }, "gpu": { "available": true, "sampleCount": 13, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 13, "peakUsedMb": 26288, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:06:28.642Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.484Z", "suiteId": "core", "scenarioId": "ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 13, "peakUsedMb": 26288, "totalMb": 31865 }, "evidenceHashes": { "result.json": "1574b4ed75c9286d12a446db4153d588449af9670a147fadea8bfa1aa36f7a02", "score.json": "7b37d4ba9c0a041ce7d5394a552edd9f93e5ff510a6292f59b4f1df34f87b5d4", "metrics.json": "92f931fb6ee01679fd3c1f045f5af2091a4e1c8537c2f1311a95bd742a662bec", "codex-inline-judge.json": "e327b38f2982053f68a802011243b6c1e63832ea706e2234115ffbfc68babf96" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab", "scenarioId": "ops-runbook-anomaly", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 21610 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "ops-runbook-anomaly", "bytes": 538, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.035616190476190475 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 12 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 21610 }, { "source": "facts", "path": "outcome.durationMs", "value": 74794 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-07-47-173Z-3vdq", "runDir": "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-07-47-173Z-3vdq", "finishedAt": "2026-10-28T08:08:18.123Z", "rawSuccess": true, "rawReason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 30949, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 107.56, "nativePrefillTokensPerSec": 1861.44, "inputTokens": 6052, "outputTokens": 1037, "peakRssMb": 10758, "gpuAvailable": true, "peakGpuUtilPercent": 87, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 30949, "derived": { "durationMs": 23516, "meanTokensPerSec": 107.56, "genTokensPerSec": 107.56, "genTokens": 1037, "promptTokensPerSec": 1861.44, "peakRssMb": 10758, "peakGpuUtilPercent": 87, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6047, "output": 37 } } }, "usage": { "available": true, "totalInputTokens": 6052, "totalOutputTokens": 1037, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6047, "todayTokensCached": 5471, "todayTokensOut": 37, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6047, "totalTokensCached": 5471, "totalTokensOut": 37, "totalCost": 0, "medianOutputTokensPerSec": 88.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 88.3, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6047, "cachedInputTokens": 5471, "outputTokens": 37, "cost": 0, "durationMs": 19792, "outputTokensPerSec": 88.34529266832233, "at": "2026-10-28T08:08:13.152Z" }, "lastUpdated": "2026-10-28T08:08:13.152Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:08:14.322Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 10758, "peakCpuPercent": 250.9, "firstAtMs": 3, "lastAtMs": 22497 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 87, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21528, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:07:47.174Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.525Z", "suiteId": "core", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-07-47-173Z-3vdq", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21528, "totalMb": 31865 }, "evidenceHashes": { "result.json": "e64f21c674d91edf75e5ca38f21a277bf3498109d7cad5db8cf4ab611613f979", "score.json": "c16082fb1ed43ace3539e6e0ed7c00ef2e19ee44e436b42104bfe16e6c84b867", "metrics.json": "2540b9f40b18d15f26713d652ff039cd2d011561639ed3536389384c862e3497", "codex-inline-judge.json": "ec651f613eb4c4213fb33162a8fdc8fef65a711f3fd518953b4ea98d4af33d37" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-07-47-173Z-3vdq", "scenarioId": "plan-and-estimate", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24860 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 2272, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.020632666666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24860 }, { "source": "facts", "path": "outcome.durationMs", "value": 30949 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-19-206Z-semw", "runDir": "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-19-206Z-semw", "finishedAt": "2026-10-28T08:08:49.247Z", "rawSuccess": true, "rawReason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 30040, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 105.4, "nativePrefillTokensPerSec": 1843.38, "inputTokens": 6066, "outputTokens": 933, "peakRssMb": 10762, "gpuAvailable": true, "peakGpuUtilPercent": 83, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 30040, "derived": { "durationMs": 23263, "meanTokensPerSec": 105.4, "genTokensPerSec": 105.4, "genTokens": 933, "promptTokensPerSec": 1843.38, "peakRssMb": 10762, "peakGpuUtilPercent": 83, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6061, "output": 39 } } }, "usage": { "available": true, "totalInputTokens": 6066, "totalOutputTokens": 933, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6061, "todayTokensCached": 5471, "todayTokensOut": 39, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6061, "totalTokensCached": 5471, "totalTokensOut": 39, "totalCost": 0, "medianOutputTokensPerSec": 68.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 68.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6061, "cachedInputTokens": 5471, "outputTokens": 39, "cost": 0, "durationMs": 19004, "outputTokensPerSec": 68.8828827848987, "at": "2026-10-28T08:08:43.450Z" }, "lastUpdated": "2026-10-28T08:08:43.450Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:08:45.406Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 10762, "peakCpuPercent": 260, "firstAtMs": 1, "lastAtMs": 22244 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 83, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21550, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:08:19.207Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.534Z", "suiteId": "core", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-19-206Z-semw", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21550, "totalMb": 31865 }, "evidenceHashes": { "result.json": "3ac25a9455aa6b5349f685efdf338c6a7e3f54aa22a59fbd2b10541d78232bf6", "score.json": "1bc1c3321d52f98660acbc8f735478c20f2ca4eafce8340cd11a37c1fc322d59", "metrics.json": "9c927d8e0b9b1ce2aaa70c73d4df4ab0b26d728b5d1b71d144ae5a6ef19f4a79", "codex-inline-judge.json": "214daf616a3d416ffe4b4063bee5fcfcc4f062043b30ed42cd97d1a68694870c" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-19-206Z-semw", "scenarioId": "plan-and-estimate", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23006 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 2426, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.02002666666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23006 }, { "source": "facts", "path": "outcome.durationMs", "value": 30040 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-50-324Z-r4y0", "runDir": "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-50-324Z-r4y0", "finishedAt": "2026-10-28T08:09:20.455Z", "rawSuccess": true, "rawReason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 30130, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 99.26, "nativePrefillTokensPerSec": 1855.47, "inputTokens": 6003, "outputTokens": 906, "peakRssMb": 10761, "gpuAvailable": true, "peakGpuUtilPercent": 85, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 30130, "derived": { "durationMs": 23259, "meanTokensPerSec": 99.26, "genTokensPerSec": 99.26, "genTokens": 906, "promptTokensPerSec": 1855.47, "peakRssMb": 10761, "peakGpuUtilPercent": 85, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6000, "output": 45 } } }, "usage": { "available": true, "totalInputTokens": 6003, "totalOutputTokens": 906, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6000, "todayTokensCached": 5477, "todayTokensOut": 45, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6000, "totalTokensCached": 5477, "totalTokensOut": 45, "totalCost": 0, "medianOutputTokensPerSec": 78, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 78, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6000, "cachedInputTokens": 5477, "outputTokens": 45, "cost": 0, "durationMs": 19197, "outputTokensPerSec": 77.97754601118268, "at": "2026-10-28T08:09:14.843Z" }, "lastUpdated": "2026-10-28T08:09:14.843Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:09:16.579Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 10761, "peakCpuPercent": 255.3, "firstAtMs": 0, "lastAtMs": 22230 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 85, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21541, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:08:50.325Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.543Z", "suiteId": "core", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-50-324Z-r4y0", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 21541, "totalMb": 31865 }, "evidenceHashes": { "result.json": "cfd4451151594fd026044a97f8b49ce5c4289530d0221690c642b47d967d97e3", "score.json": "256975e011fda59dbbb9752185b686639350434ae2ff4211efe10e663c31dbf8", "metrics.json": "8f75cc969e73d37c8a02405af4249427f4be722166ee37b9e33ad9d4771b073c", "codex-inline-judge.json": "466e693dc1ae97928eee03449d8c7d4ee1313b94db6dbe166e82db9d3c308385" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-50-324Z-r4y0", "scenarioId": "plan-and-estimate", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23366 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 2126, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.020086666666666666 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23366 }, { "source": "facts", "path": "outcome.durationMs", "value": 30130 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-09-24-593Z-7k5u", "runDir": "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-09-24-593Z-7k5u", "finishedAt": "2026-10-28T08:10:00.691Z", "rawSuccess": true, "rawReason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 36097, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 114, "nativePrefillTokensPerSec": 1972.3, "inputTokens": 11146, "outputTokens": 1132, "peakRssMb": 10754, "gpuAvailable": true, "peakGpuUtilPercent": 89, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 36097, "derived": { "durationMs": 28942, "meanTokensPerSec": 114, "genTokensPerSec": 114, "genTokens": 1132, "promptTokensPerSec": 1972.3, "peakRssMb": 10754, "peakGpuUtilPercent": 89, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 11143, "output": 310 } } }, "usage": { "available": true, "totalInputTokens": 11146, "totalOutputTokens": 1132, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11143, "todayTokensCached": 10666, "todayTokensOut": 310, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11143, "totalTokensCached": 10666, "totalTokensOut": 310, "totalCost": 0, "medianOutputTokensPerSec": 104.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 104.1, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11143, "cachedInputTokens": 10666, "outputTokens": 310, "cost": 0, "durationMs": 22606, "outputTokensPerSec": 104.12833463409505, "at": "2026-10-28T08:09:52.506Z" }, "lastUpdated": "2026-10-28T08:09:52.506Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:09:56.468Z" } }, "source": "engine-log" }, "process": { "sampleCount": 6, "peakRssMb": 10754, "peakCpuPercent": 244.5, "firstAtMs": 0, "lastAtMs": 27887 }, "gpu": { "available": true, "sampleCount": 6, "peakUtilPercent": 89, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 21552, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:09:24.594Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.364Z", "suiteId": "core", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-09-24-593Z-7k5u", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 21552, "totalMb": 31865 }, "evidenceHashes": { "result.json": "2194c9e3af39874d016ef44f41867d9c7eff159ad3026485f0075d34506a210c", "score.json": "bd47e5a160906518dd1eedfab58bbf063d497ec737b1a46168a3332bb0cb7c38", "metrics.json": "acdf0e958d72dca166c7f1babb24f469387ddad0bedc82f9a41e34e9f7245eea", "codex-inline-judge.json": "178a43d7d72bde5cd9c9fd8a7e68adf319330a89ca6d4ba0175c059ffb612a44" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-09-24-593Z-7k5u", "scenarioId": "conflict-synthesis", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24268 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 1494, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.02005388888888889 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24268 }, { "source": "facts", "path": "outcome.durationMs", "value": 36097 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-01-356Z-w8hn", "runDir": "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-01-356Z-w8hn", "finishedAt": "2026-10-28T08:10:37.083Z", "rawSuccess": true, "rawReason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 35726, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 111.21, "nativePrefillTokensPerSec": 1962.24, "inputTokens": 11109, "outputTokens": 949, "peakRssMb": 10770, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 35726, "derived": { "durationMs": 28832, "meanTokensPerSec": 111.21, "genTokensPerSec": 111.21, "genTokens": 949, "promptTokensPerSec": 1962.24, "peakRssMb": 10770, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 11106, "output": 88 } } }, "usage": { "available": true, "totalInputTokens": 11109, "totalOutputTokens": 949, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11106, "todayTokensCached": 10655, "todayTokensOut": 88, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11106, "totalTokensCached": 10655, "totalTokensOut": 88, "totalCost": 0, "medianOutputTokensPerSec": 104.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 104.8, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11106, "cachedInputTokens": 10655, "outputTokens": 88, "cost": 0, "durationMs": 21672, "outputTokensPerSec": 104.81220544156056, "at": "2026-10-28T08:10:28.356Z" }, "lastUpdated": "2026-10-28T08:10:28.356Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:10:33.143Z" } }, "source": "engine-log" }, "process": { "sampleCount": 6, "peakRssMb": 10770, "peakCpuPercent": 261.1, "firstAtMs": 2, "lastAtMs": 27787 }, "gpu": { "available": true, "sampleCount": 6, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 21614, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:10:01.357Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.374Z", "suiteId": "core", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-01-356Z-w8hn", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 21614, "totalMb": 31865 }, "evidenceHashes": { "result.json": "edd9b2c4d36b59a2166e5e6bfc00a75dde6e38b762e869bff387f3a908d3e63c", "score.json": "c4d642614ffd1c2cc9e8386644f44ab3fe2006ae496de8496543bb39a0fa3584", "metrics.json": "031175035859387062578184f2bd08cde12b5c33b960e92a2bc71acf65495870", "codex-inline-judge.json": "53f588abcfac3d91ac0bb1ca91434ace38f6bd281026a4d7e3afb6195ab33eb8" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-01-356Z-w8hn", "scenarioId": "conflict-synthesis", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25576 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 1517, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.019847777777777777 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25576 }, { "source": "facts", "path": "outcome.durationMs", "value": 35726 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "core", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-38-043Z-rol3", "runDir": "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-38-043Z-rol3", "finishedAt": "2026-10-28T08:11:13.816Z", "rawSuccess": true, "rawReason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 35771, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 119.85, "nativePrefillTokensPerSec": 1958.83, "inputTokens": 11093, "outputTokens": 987, "peakRssMb": 10756, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 35771, "derived": { "durationMs": 28819, "meanTokensPerSec": 119.85, "genTokensPerSec": 119.85, "genTokens": 987, "promptTokensPerSec": 1958.83, "peakRssMb": 10756, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 11090, "output": 114 } } }, "usage": { "available": true, "totalInputTokens": 11093, "totalOutputTokens": 987, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11090, "todayTokensCached": 10655, "todayTokensOut": 114, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11090, "totalTokensCached": 10655, "totalTokensOut": 114, "totalCost": 0, "medianOutputTokensPerSec": 122.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 122.2, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11090, "cachedInputTokens": 10655, "outputTokens": 114, "cost": 0, "durationMs": 20849, "outputTokensPerSec": 122.21580499116368, "at": "2026-10-28T08:11:04.122Z" }, "lastUpdated": "2026-10-28T08:11:04.122Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:11:09.782Z" } }, "source": "engine-log" }, "process": { "sampleCount": 6, "peakRssMb": 10756, "peakCpuPercent": 246.3, "firstAtMs": 0, "lastAtMs": 27771 }, "gpu": { "available": true, "sampleCount": 6, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 21552, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:10:38.045Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.382Z", "suiteId": "core", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-38-043Z-rol3", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 21552, "totalMb": 31865 }, "evidenceHashes": { "result.json": "a51ff77c870e35efda77d093d821abd2b05d2ddcc8b5fdc373e0c36b8bb4c6ee", "score.json": "0888f4552d06b6e0d1bffb8b18aa50f7e6efab00a36e1eb38bb41e295f08b4a0", "metrics.json": "6413acd269cb7c756dde5a92c5a4023d83e9c35f786e8ddc3571dfb6fa638352", "codex-inline-judge.json": "c518c3e2cf18ba066d49349806505db25e94b767b629b7b9a4c985bf81fddd0f" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-38-043Z-rol3", "scenarioId": "conflict-synthesis", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24572 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 1417, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.019872777777777777 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24572 }, { "source": "facts", "path": "outcome.durationMs", "value": 35771 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "constrained-comms", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4", "runDir": "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4", "finishedAt": "2026-10-28T08:11:49.912Z", "rawSuccess": true, "rawReason": "notice satisfies the full constraint stack (signals: word-band, payment-data-disclosure, support-contact, service-credit, no-guarantee, no-never-again, no-100-secure, no-unhackable, grounded-facts)", "durationMs": 29096, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 142.22, "nativePrefillTokensPerSec": 2087.06, "inputTokens": 15530, "outputTokens": 459, "peakRssMb": 10909, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 29096, "derived": { "durationMs": 23277, "meanTokensPerSec": 142.22, "genTokensPerSec": 142.22, "genTokens": 459, "promptTokensPerSec": 2087.06, "peakRssMb": 10909, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 5606, "output": 201 } } }, "usage": { "available": true, "totalInputTokens": 15530, "totalOutputTokens": 459, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 5606, "todayTokensCached": 0, "todayTokensOut": 201, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 5606, "totalTokensCached": 0, "totalTokensOut": 201, "totalCost": 0, "medianOutputTokensPerSec": 119.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 119.8, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5606, "outputTokens": 201, "cost": 0, "durationMs": 18034, "outputTokensPerSec": 119.76456681455593, "at": "2026-10-28T08:11:44.356Z" }, "lastUpdated": "2026-10-28T08:11:44.356Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:11:47.153Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 10909, "peakCpuPercent": 220.4, "firstAtMs": 8, "lastAtMs": 22229 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22020, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:11:20.816Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.661Z", "suiteId": "productivity", "scenarioId": "constrained-comms", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22020, "totalMb": 31865 }, "evidenceHashes": { "result.json": "38f54aea34e8e09d4eb8451b1f9615aed6514b499ed4faf152f65c896e69265b", "score.json": "8b3199e448b17aa10d4616dfc94c6b4e9c59cd5bb40a198c698592e716004f71", "metrics.json": "daf9ea112ee48f8baafd2643d22b62860af4bf516caa2833a76d1eb5e27793b0", "codex-inline-judge.json": "e1bb8e5a48d097ca4d437f7cb29f5ab5f1138478262ea514ea4f4bbd90e305f3" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4", "scenarioId": "constrained-comms", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23518 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 885, "score": 9, "scoreMax": null, "signals": [ "word-band", "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.024246666666666666 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23518 }, { "source": "facts", "path": "outcome.durationMs", "value": 29096 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "constrained-comms", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8", "runDir": "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8", "finishedAt": "2026-10-28T08:12:19.934Z", "rawSuccess": true, "rawReason": "notice satisfies the full constraint stack (signals: word-band, payment-data-disclosure, support-contact, service-credit, no-guarantee, no-never-again, no-100-secure, no-unhackable, grounded-facts)", "durationMs": 29051, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 127.21, "nativePrefillTokensPerSec": 2083.64, "inputTokens": 15528, "outputTokens": 605, "peakRssMb": 10911, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 29051, "derived": { "durationMs": 23386, "meanTokensPerSec": 127.21, "genTokensPerSec": 127.21, "genTokens": 605, "promptTokensPerSec": 2083.64, "peakRssMb": 10911, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 5605, "output": 238 } } }, "usage": { "available": true, "totalInputTokens": 15528, "totalOutputTokens": 605, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 5605, "todayTokensCached": 0, "todayTokensOut": 238, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 5605, "totalTokensCached": 0, "totalTokensOut": 238, "totalCost": 0, "medianOutputTokensPerSec": 112.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 112.3, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5605, "outputTokens": 238, "cost": 0, "durationMs": 19512, "outputTokensPerSec": 112.3289826785865, "at": "2026-10-28T08:12:16.272Z" }, "lastUpdated": "2026-10-28T08:12:16.272Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:12:17.246Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 10911, "peakCpuPercent": 242.2, "firstAtMs": 0, "lastAtMs": 22310 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22037, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:11:50.883Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.670Z", "suiteId": "productivity", "scenarioId": "constrained-comms", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22037, "totalMb": 31865 }, "evidenceHashes": { "result.json": "012ae9e6e7a499c26bac97746bc5bf7720729a2f7864d373c727a7bfaf70f631", "score.json": "f94a96c4222da19bf8acd656637c9bcf5036dccc04c2337911a2bd25fdcd550f", "metrics.json": "276892ce648305e766d705ddf9a2e0755034f579796e66dbaab6893866045383", "codex-inline-judge.json": "2f360d9123394739b1bb4d966976e65fc2150875125cf9f439d717e30748d633" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8", "scenarioId": "constrained-comms", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25379 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 1025, "score": 9, "scoreMax": null, "signals": [ "word-band", "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.024209166666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25379 }, { "source": "facts", "path": "outcome.durationMs", "value": 29051 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "constrained-comms", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho", "runDir": "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho", "finishedAt": "2026-10-28T08:13:02.222Z", "rawSuccess": true, "rawReason": "notice satisfies the full constraint stack (signals: word-band, payment-data-disclosure, support-contact, service-credit, no-guarantee, no-never-again, no-100-secure, no-unhackable, grounded-facts)", "durationMs": 41260, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 136.68, "nativePrefillTokensPerSec": 2039.81, "inputTokens": 22034, "outputTokens": 973, "peakRssMb": 13267, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 41260, "derived": { "durationMs": 34475, "meanTokensPerSec": 136.68, "genTokensPerSec": 136.68, "genTokens": 973, "promptTokensPerSec": 2039.81, "peakRssMb": 13267, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 12111, "output": 422 } } }, "usage": { "available": true, "totalInputTokens": 22034, "totalOutputTokens": 973, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 12111, "todayTokensCached": 6247, "todayTokensOut": 422, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 12111, "totalTokensCached": 6247, "totalTokensOut": 422, "totalCost": 0, "medianOutputTokensPerSec": 124.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 124.6, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6506, "cachedInputTokens": 6247, "outputTokens": 232, "cost": 0, "durationMs": 7543, "outputTokensPerSec": 140.73875049578425, "at": "2026-10-28T08:12:55.404Z" }, "lastUpdated": "2026-10-28T08:12:55.404Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:12:58.421Z" } }, "source": "engine-log" }, "process": { "sampleCount": 7, "peakRssMb": 13267, "peakCpuPercent": 224.2, "firstAtMs": 1, "lastAtMs": 33415 }, "gpu": { "available": true, "sampleCount": 7, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 24393, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:12:20.962Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:14:06.678Z", "suiteId": "productivity", "scenarioId": "constrained-comms", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 7, "peakUsedMb": 24393, "totalMb": 31865 }, "evidenceHashes": { "result.json": "ba2714fe606304843bc1ed8c85e639bf970ebd3c9807cef12206ef327fe1dcc1", "score.json": "3a805034f06a241be38f5bcfc763ec9677b261af50f94aadc0927426bdaaca25", "metrics.json": "565f01b73a7feb1d3dcffb71e2017316979a62a89f070b206f45a25d2449342e", "codex-inline-judge.json": "f278384821e555aed9e3e2daf4a67350eef019d9377d14a85f0f99764cbbfdb5" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho", "scenarioId": "constrained-comms", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23787 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 1038, "score": 9, "scoreMax": null, "signals": [ "word-band", "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.034383333333333335 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23787 }, { "source": "facts", "path": "outcome.durationMs", "value": 41260 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "records-intake", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho", "runDir": "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho", "finishedAt": "2026-10-28T08:15:16.756Z", "rawSuccess": true, "rawReason": "attendees.csv matches the declared schema with all golden rows (signals: csv-shape, record-schema, golden-rows)", "durationMs": 130374, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 137.57, "nativePrefillTokensPerSec": 1841.68, "inputTokens": 16926, "outputTokens": 6483, "peakRssMb": 15103, "gpuAvailable": true, "peakGpuUtilPercent": 91, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 130374, "derived": { "durationMs": 123408, "meanTokensPerSec": 137.57, "genTokensPerSec": 137.57, "genTokens": 6483, "promptTokensPerSec": 1841.68, "peakRssMb": 15103, "peakGpuUtilPercent": 91, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 16900, "output": 271 } } }, "usage": { "available": true, "totalInputTokens": 16926, "totalOutputTokens": 6483, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 16900, "todayTokensCached": 14980, "todayTokensOut": 271, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 16900, "totalTokensCached": 14980, "totalTokensOut": 271, "totalCost": 0, "medianOutputTokensPerSec": 103.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 103.3, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7559, "cachedInputTokens": 6993, "outputTokens": 47, "cost": 0, "durationMs": 23268, "outputTokensPerSec": 90.82311408765221, "at": "2026-10-28T08:15:08.822Z" }, "lastUpdated": "2026-10-28T08:15:08.822Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:15:12.852Z" } }, "source": "engine-log" }, "process": { "sampleCount": 23, "peakRssMb": 15103, "peakCpuPercent": 269.3, "firstAtMs": 1, "lastAtMs": 122347 }, "gpu": { "available": true, "sampleCount": 23, "peakUtilPercent": 91, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 26479, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:13:06.382Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:25:07.561Z", "suiteId": "productivity", "scenarioId": "records-intake", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 26479, "totalMb": 31865 }, "evidenceHashes": { "result.json": "0243390400d0d116144e8069a07ec06525b70af60aa4d55de49f42c5b6b03117", "score.json": "5778bff591fcafec2311b33f329a56b53b0c295e48700a48221158c56ae010bf", "metrics.json": "b54830c0cadef5ec2d3708960a8db7065072d5e96f07a5a586d1dfe507c16578", "codex-inline-judge.json": "1d13130dac6238e3d4f1140a4cde23a374eaa0ca728269ba3142a262cb2045dc" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho", "scenarioId": "records-intake", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34223 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "records-intake", "bytes": 1132, "score": 3, "scoreMax": 3, "signals": [ "csv-shape", "record-schema", "golden-rows" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.086916 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 6 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34223 }, { "source": "facts", "path": "outcome.durationMs", "value": 130374 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "records-intake", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i", "runDir": "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i", "finishedAt": "2026-10-28T08:17:27.674Z", "rawSuccess": true, "rawReason": "attendees.csv matches the declared schema with all golden rows (signals: csv-shape, record-schema, golden-rows)", "durationMs": 129980, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 138.8, "nativePrefillTokensPerSec": 1838.34, "inputTokens": 16361, "outputTokens": 6100, "peakRssMb": 14466, "gpuAvailable": true, "peakGpuUtilPercent": 97, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 129980, "derived": { "durationMs": 123321, "meanTokensPerSec": 138.8, "genTokensPerSec": 138.8, "genTokens": 6100, "promptTokensPerSec": 1838.34, "peakRssMb": 14466, "peakGpuUtilPercent": 97, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 16340, "output": 131 } } }, "usage": { "available": true, "totalInputTokens": 16361, "totalOutputTokens": 6100, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 16340, "todayTokensCached": 15742, "todayTokensOut": 131, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 16340, "totalTokensCached": 15742, "totalTokensOut": 131, "totalCost": 0, "medianOutputTokensPerSec": 97.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 97.8, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7485, "cachedInputTokens": 6919, "outputTokens": 45, "cost": 0, "durationMs": 23630, "outputTokensPerSec": 87.23529540051905, "at": "2026-10-28T08:17:20.430Z" }, "lastUpdated": "2026-10-28T08:17:20.430Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:17:23.991Z" } }, "source": "engine-log" }, "process": { "sampleCount": 23, "peakRssMb": 14466, "peakCpuPercent": 271.3, "firstAtMs": 3, "lastAtMs": 122261 }, "gpu": { "available": true, "sampleCount": 23, "peakUtilPercent": 97, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 25897, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:15:17.694Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:25:07.571Z", "suiteId": "productivity", "scenarioId": "records-intake", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 25897, "totalMb": 31865 }, "evidenceHashes": { "result.json": "7b06638af2fc19ec7cfac21d2840177f5732ec561535ae1a72914ec0aab3b556", "score.json": "7155ee64d3efbf6c523fa017be2b24b870d107ebe216c757522a5abb8e219bcc", "metrics.json": "ce227fe70cb81e8e029ce097ca487d7d4f63bb07d0e72436fe7e4300068892f0", "codex-inline-judge.json": "43b63a2d00e36f266a8977ee8bc2499902eb660702e4641387b67740b0e4f234" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i", "scenarioId": "records-intake", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 40478 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "records-intake", "bytes": 1132, "score": 3, "scoreMax": 3, "signals": [ "csv-shape", "record-schema", "golden-rows" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08665333333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 5 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 40478 }, { "source": "facts", "path": "outcome.durationMs", "value": 129980 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "records-intake", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu", "runDir": "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu", "finishedAt": "2026-10-28T08:19:38.496Z", "rawSuccess": true, "rawReason": "attendees.csv matches the declared schema with all golden rows (signals: csv-shape, record-schema, golden-rows)", "durationMs": 129686, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 143.1, "nativePrefillTokensPerSec": 1824.91, "inputTokens": 16947, "outputTokens": 6433, "peakRssMb": 15102, "gpuAvailable": true, "peakGpuUtilPercent": 98, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 129686, "derived": { "durationMs": 123044, "meanTokensPerSec": 143.1, "genTokensPerSec": 143.1, "genTokens": 6433, "promptTokensPerSec": 1824.91, "peakRssMb": 15102, "peakGpuUtilPercent": 98, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 16926, "output": 284 } } }, "usage": { "available": true, "totalInputTokens": 16947, "totalOutputTokens": 6433, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 16926, "todayTokensCached": 15165, "todayTokensOut": 284, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 16926, "totalTokensCached": 15165, "totalTokensOut": 284, "totalCost": 0, "medianOutputTokensPerSec": 108.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 108.9, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7579, "cachedInputTokens": 7013, "outputTokens": 40, "cost": 0, "durationMs": 23818, "outputTokensPerSec": 89.73459268548025, "at": "2026-10-28T08:19:31.344Z" }, "lastUpdated": "2026-10-28T08:19:31.344Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:19:34.767Z" } }, "source": "engine-log" }, "process": { "sampleCount": 23, "peakRssMb": 15102, "peakCpuPercent": 268.1, "firstAtMs": 1, "lastAtMs": 121944 }, "gpu": { "available": true, "sampleCount": 23, "peakUtilPercent": 98, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 26555, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:17:28.810Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:25:07.580Z", "suiteId": "productivity", "scenarioId": "records-intake", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 26555, "totalMb": 31865 }, "evidenceHashes": { "result.json": "62560fb737cb114069e80e9454f8c1e420e9d8f0eb5a6ce64561b2c6cfb1bf63", "score.json": "08c46d5dd1dc38b0d1adad0a53146f063eea9d22d659393221e1a7ed2c9e27a5", "metrics.json": "204e516a06158261e0c50a33e485659bd7b526ae34e658f93b6f61d1d34b9e28", "codex-inline-judge.json": "e623b0a062633ecfd62c86914dbb471912aa3602635b8bdabf497f0b0db34f29" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu", "scenarioId": "records-intake", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 32357 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "records-intake", "bytes": 1132, "score": 3, "scoreMax": 3, "signals": [ "csv-shape", "record-schema", "golden-rows" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08645733333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 6 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 32357 }, { "source": "facts", "path": "outcome.durationMs", "value": 129686 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri", "runDir": "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri", "finishedAt": "2026-10-28T08:20:12.890Z", "rawSuccess": true, "rawReason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 30185, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 100.95, "nativePrefillTokensPerSec": 1830.06, "inputTokens": 6042, "outputTokens": 1021, "peakRssMb": 10793, "gpuAvailable": true, "peakGpuUtilPercent": 85, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 30185, "derived": { "durationMs": 23322, "meanTokensPerSec": 100.95, "genTokensPerSec": 100.95, "genTokens": 1021, "promptTokensPerSec": 1830.06, "peakRssMb": 10793, "peakGpuUtilPercent": 85, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6037, "output": 42 } } }, "usage": { "available": true, "totalInputTokens": 6042, "totalOutputTokens": 1021, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6037, "todayTokensCached": 5471, "todayTokensOut": 42, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6037, "totalTokensCached": 5471, "totalTokensOut": 42, "totalCost": 0, "medianOutputTokensPerSec": 77, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 77, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6037, "cachedInputTokens": 5471, "outputTokens": 42, "cost": 0, "durationMs": 20271, "outputTokensPerSec": 76.98171590367146, "at": "2026-10-28T08:20:08.230Z" }, "lastUpdated": "2026-10-28T08:20:08.230Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:20:08.962Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 10793, "peakCpuPercent": 264.5, "firstAtMs": 3, "lastAtMs": 22231 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 85, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22191, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:19:42.705Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:25:07.535Z", "suiteId": "productivity", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22191, "totalMb": 31865 }, "evidenceHashes": { "result.json": "c027913aefd828be739a699c57ad4ce251a1cb67c2eb0ca8c3ca6ab4ddcd63e3", "score.json": "5aed3486d7d79f7283059cdaa2a98b93fc9121a55a353c0ce2bbe9a29b23dc08", "metrics.json": "5563f642a23a2e15c717907678661463ae9d77b1693d38d24def169158363c33", "codex-inline-judge.json": "201d08eb64aa52e52f67bfb3e60cec8c36ff82ab4684b852002a9f4b2e10ff43" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri", "scenarioId": "plan-and-estimate", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24307 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 2255, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.020123333333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24307 }, { "source": "facts", "path": "outcome.durationMs", "value": 30185 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m", "runDir": "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m", "finishedAt": "2026-10-28T08:20:44.137Z", "rawSuccess": true, "rawReason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 30287, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 113.94, "nativePrefillTokensPerSec": 1883.21, "inputTokens": 5974, "outputTokens": 932, "peakRssMb": 10750, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 30287, "derived": { "durationMs": 23320, "meanTokensPerSec": 113.94, "genTokensPerSec": 113.94, "genTokens": 932, "promptTokensPerSec": 1883.21, "peakRssMb": 10750, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 5971, "output": 49 } } }, "usage": { "available": true, "totalInputTokens": 5974, "totalOutputTokens": 932, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 5971, "todayTokensCached": 5473, "todayTokensOut": 49, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 5971, "totalTokensCached": 5473, "totalTokensOut": 49, "totalCost": 0, "medianOutputTokensPerSec": 101.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 101.3, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 5971, "cachedInputTokens": 5473, "outputTokens": 49, "cost": 0, "durationMs": 18645, "outputTokensPerSec": 101.2942450096757, "at": "2026-10-28T08:20:37.883Z" }, "lastUpdated": "2026-10-28T08:20:37.883Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:20:40.185Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 10750, "peakCpuPercent": 257.3, "firstAtMs": 1, "lastAtMs": 22292 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22225, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:20:13.850Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:25:07.544Z", "suiteId": "productivity", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22225, "totalMb": 31865 }, "evidenceHashes": { "result.json": "0700d51419badecb39272770561aa8652ee9de546257cbe043859e7addd503ff", "score.json": "0d7b74f7effe607ac28a7145cd06e9e7ad2d3900d4e2cb693423edda6512d33b", "metrics.json": "957896a4081cb98fd0d117abd6d0d6a71e26fa168931dc162bb56381f4c89b0e", "codex-inline-judge.json": "8485c9b1f9b3f879da1c7bbbe898eca86dc69eb6b29c149d5ff398d32fcabe04" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m", "scenarioId": "plan-and-estimate", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23021 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 1965, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.020191333333333332 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23021 }, { "source": "facts", "path": "outcome.durationMs", "value": 30287 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h", "runDir": "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h", "finishedAt": "2026-10-28T08:21:15.191Z", "rawSuccess": true, "rawReason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 30122, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 111.68, "nativePrefillTokensPerSec": 1853.03, "inputTokens": 6043, "outputTokens": 979, "peakRssMb": 10762, "gpuAvailable": true, "peakGpuUtilPercent": 88, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 30122, "derived": { "durationMs": 23269, "meanTokensPerSec": 111.68, "genTokensPerSec": 111.68, "genTokens": 979, "promptTokensPerSec": 1853.03, "peakRssMb": 10762, "peakGpuUtilPercent": 88, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6038, "output": 54 } } }, "usage": { "available": true, "totalInputTokens": 6043, "totalOutputTokens": 979, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 6038, "todayTokensCached": 5471, "todayTokensOut": 54, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 6038, "totalTokensCached": 5471, "totalTokensOut": 54, "totalCost": 0, "medianOutputTokensPerSec": 126.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 126.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6038, "cachedInputTokens": 5471, "outputTokens": 54, "cost": 0, "durationMs": 19356, "outputTokensPerSec": 126.88107173807917, "at": "2026-10-28T08:21:09.796Z" }, "lastUpdated": "2026-10-28T08:21:09.796Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:21:11.317Z" } }, "source": "engine-log" }, "process": { "sampleCount": 5, "peakRssMb": 10762, "peakCpuPercent": 248.3, "firstAtMs": 3, "lastAtMs": 22263 }, "gpu": { "available": true, "sampleCount": 5, "peakUtilPercent": 88, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22225, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:20:45.069Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:25:07.551Z", "suiteId": "productivity", "scenarioId": "plan-and-estimate", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 5, "peakUsedMb": 22225, "totalMb": 31865 }, "evidenceHashes": { "result.json": "5e62c5a77a6c26e01b559e38ab0b425605793390d12aed7b7c3b81b053f8a5b8", "score.json": "0da3e6a34e1fc42dc90678517f3277669e893cda4728da0b5d8bfde645cc9229", "metrics.json": "9d580412bfa2ff14e0e4a81be4939250aca48fd58e9845716543fb15e2c62b45", "codex-inline-judge.json": "556fb2276e1755fd282a5bd7351665c05402f46fd1fea343c4cb196c1601c0a2" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h", "scenarioId": "plan-and-estimate", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23613 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 2200, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.020081333333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23613 }, { "source": "facts", "path": "outcome.durationMs", "value": 30122 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "meeting-followup", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho", "runDir": "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho", "finishedAt": "2026-10-28T08:24:11.615Z", "rawSuccess": true, "rawReason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "durationMs": 172278, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.5, "judgeArtifactCandidates": [ "workspace/orchid-launch-follow-up/meeting-brief.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 130.68, "nativePrefillTokensPerSec": 1862.5, "inputTokens": 24527, "outputTokens": 4369, "peakRssMb": 17394, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 172278, "derived": { "durationMs": 161715, "meanTokensPerSec": 130.68, "genTokensPerSec": 130.68, "genTokens": 4369, "promptTokensPerSec": 1862.5, "peakRssMb": 17394, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44339, "output": 2400 } } }, "usage": { "available": true, "totalInputTokens": 24527, "totalOutputTokens": 4369, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 44339, "todayTokensCached": 38196, "todayTokensOut": 2400, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 44339, "totalTokensCached": 38196, "totalTokensOut": 2400, "totalCost": 0, "medianOutputTokensPerSec": 135.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 135.5, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11308, "cachedInputTokens": 10973, "outputTokens": 63, "cost": 0, "durationMs": 4092, "outputTokensPerSec": 150.05711851608032, "at": "2026-10-28T08:24:02.171Z" }, "lastUpdated": "2026-10-28T08:24:02.171Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:24:04.045Z" } }, "source": "engine-log" }, "process": { "sampleCount": 30, "peakRssMb": 17394, "peakCpuPercent": 263.6, "firstAtMs": 1, "lastAtMs": 160671 }, "gpu": { "available": true, "sampleCount": 30, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 28783, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:21:19.337Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T08:26:06.213Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 883-byte meeting brief read against the original kickoff, authoritative transcript, roster, stale agenda and exact scenario judge axes captured in judge-plan.json. The CSV is outside this declared artifact judgment; its deterministic gate is separate.", "suiteId": "productivity", "scenarioId": "meeting-followup", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho", "artifactPath": "workspace/orchid-launch-follow-up/meeting-brief.md", "artifactKind": "markdown", "scoreAxes": { "decisionClarity": 6, "grounding": 7, "actionability": 6, "brevity": 7 }, "justification": "Clearly separates the three decisions, four named owners and due dates, unresolved annual-plan and rollback questions, and legal-review risk. Rejects the stale global/rolling proposals and preserves the field-map dependency. Compact and easy to scan, though actions lack IDs and the prerequisite sits only in the final section rather than beside Priya's action.", "artifactSha256": "a478d5fd8377e3f898204429254132d650c199eb563c3e074cad0f1884d5a2bd", "inputBytes": 883, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.5 }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 28783, "totalMb": 31865 }, "evidenceHashes": { "result.json": "d2a1b6929ce9b5302368eff8c8bb2d1af54c36f4b57e454a590b1e18b1d128a0", "score.json": "c7732788d53f3c8949c0c81eab5a9ffa2b8b65bc3d2d41d4877f8f0cceca0c38", "metrics.json": "071551f76f998b5ff77cbfae842dd50fa3eacd048100c19ddb01d0e6d4a8c0ea", "codex-inline-judge.json": "a1099d66abb9cb577d9335448294302d6a0eefbe8316780a9c071d63f44820f5" }, "rubricId": "meeting-followup", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho", "scenarioId": "meeting-followup", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24985 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "meeting-followup", "bytes": 1155, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.09571 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24985 }, { "source": "facts", "path": "outcome.durationMs", "value": 172278 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "meeting-followup", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9", "runDir": "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9", "finishedAt": "2026-10-28T08:26:39.045Z", "rawSuccess": true, "rawReason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "durationMs": 146214, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.5, "judgeArtifactCandidates": [ "workspace/orchid-launch-follow-up/meeting-brief.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 139.08, "nativePrefillTokensPerSec": 1985.37, "inputTokens": 22321, "outputTokens": 2474, "peakRssMb": 14175, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 146214, "derived": { "durationMs": 139549, "meanTokensPerSec": 139.08, "genTokensPerSec": 139.08, "genTokens": 2474, "promptTokensPerSec": 1985.37, "peakRssMb": 14175, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22316, "output": 1470 } } }, "usage": { "available": true, "totalInputTokens": 22321, "totalOutputTokens": 2474, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 22316, "todayTokensCached": 8500, "todayTokensOut": 1470, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 22316, "totalTokensCached": 8500, "totalTokensOut": 1470, "totalCost": 0, "medianOutputTokensPerSec": 135.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 135.1, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9104, "outputTokens": 1176, "cost": 0, "durationMs": 13746, "outputTokensPerSec": 135.07187893145382, "at": "2026-10-28T08:26:32.407Z" }, "lastUpdated": "2026-10-28T08:26:32.407Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:26:35.781Z" } }, "source": "engine-log" }, "process": { "sampleCount": 26, "peakRssMb": 14175, "peakCpuPercent": 254.3, "firstAtMs": 0, "lastAtMs": 138493 }, "gpu": { "available": true, "sampleCount": 26, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 25511, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:24:12.831Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T08:32:11.445Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 889-byte brief read against the kickoff, authoritative transcript, roster, stale agenda and exact scenario axes in judge-plan.json. CSV correctness remains a separate deterministic gate.", "suiteId": "productivity", "scenarioId": "meeting-followup", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9", "artifactPath": "workspace/orchid-launch-follow-up/meeting-brief.md", "artifactKind": "markdown", "scoreAxes": { "decisionClarity": 6, "grounding": 7, "actionability": 6, "brevity": 7 }, "justification": "Faithfully records September 14, EU-only scope and the overnight batch, with all four roster owners and deadlines. Keeps annual-plan scope and rollback ownership unresolved and names the unscheduled legal review. Four short sections are easy to scan; action IDs are absent and Priya's prerequisite appears only in the final section.", "artifactSha256": "e43118cfd1c08721ae906cd58f3a9037c0e4b5b12547a9b00f3f41a521bb8664", "inputBytes": 889, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.5 }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 25511, "totalMb": 31865 }, "evidenceHashes": { "result.json": "f2ad0b056a8cc8ef372eb9add98448a659f0755b9553739db4112ddf07f51b02", "score.json": "056294a94f9bcba11dab1ed88d259f3a274451eb01303ce2327329bdc7a8ad00", "metrics.json": "c54c58063a3c812c863441b4c2bfdc522f650988397ea7c2613b30a2613c9953", "codex-inline-judge.json": "3e0c9e8364524ff197b73c0771a91dc22fec2eb12a6c4a6b70c0db857847f93f" }, "rubricId": "meeting-followup", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9", "scenarioId": "meeting-followup", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25034 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "meeting-followup", "bytes": 1161, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08123 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25034 }, { "source": "facts", "path": "outcome.durationMs", "value": 146214 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "meeting-followup", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7", "runDir": "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7", "finishedAt": "2026-10-28T08:29:29.149Z", "rawSuccess": true, "rawReason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "durationMs": 169601, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.25, "judgeArtifactCandidates": [ "workspace/orchid-launch-follow-up/meeting-brief.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 130.36, "nativePrefillTokensPerSec": 1927.62, "inputTokens": 23329, "outputTokens": 4703, "peakRssMb": 15824, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 169601, "derived": { "durationMs": 161734, "meanTokensPerSec": 130.36, "genTokensPerSec": 130.36, "genTokens": 4703, "promptTokensPerSec": 1927.62, "peakRssMb": 15824, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 32784, "output": 2701 } } }, "usage": { "available": true, "totalInputTokens": 23329, "totalOutputTokens": 4703, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 32784, "todayTokensCached": 27081, "todayTokensOut": 2701, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 32784, "totalTokensCached": 27081, "totalTokensOut": 2701, "totalCost": 0, "medianOutputTokensPerSec": 135.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 135.4, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10095, "cachedInputTokens": 9465, "outputTokens": 2334, "cost": 0, "durationMs": 18016, "outputTokensPerSec": 135.70924367254923, "at": "2026-10-28T08:29:19.395Z" }, "lastUpdated": "2026-10-28T08:29:19.395Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:29:24.166Z" } }, "source": "engine-log" }, "process": { "sampleCount": 30, "peakRssMb": 15824, "peakCpuPercent": 262.6, "firstAtMs": 1, "lastAtMs": 160691 }, "gpu": { "available": true, "sampleCount": 30, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 27143, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:26:39.548Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T08:32:11.450Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 948-byte brief read against the kickoff, authoritative transcript, roster, stale agenda and exact scenario axes in judge-plan.json. CSV correctness remains a separate deterministic gate.", "suiteId": "productivity", "scenarioId": "meeting-followup", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7", "artifactPath": "workspace/orchid-launch-follow-up/meeting-brief.md", "artifactKind": "markdown", "scoreAxes": { "decisionClarity": 6, "grounding": 6, "actionability": 6, "brevity": 7 }, "justification": "Clear, compact sections preserve the launch decisions, four owners and dates, open questions and legal risk. Priya's dependency is usefully beside her action as well as in the dependency section. Jordan's action shifts slightly from scheduling a decision review to reviewing the decision by that date; action IDs are omitted.", "artifactSha256": "77dcd0b6c4a4f8aa01577c082a58d9ce92597e2f10df90d6a05403fe9761ab4c", "inputBytes": 948, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.25 }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 27143, "totalMb": 31865 }, "evidenceHashes": { "result.json": "9fbb829e9d366b39888a66c038a15e4e9adc9b0ee10a2ee3a69c365c205bf5d9", "score.json": "24f1db6fe4439e53ac4773c42d59cfcf097cd122f10c8f3b3fbf4af8ca3be6e2", "metrics.json": "50bfe7eabdb4b7cb9c6466f8eff8362bacdfd6efe3cd33d63406f9b6a53b9351", "codex-inline-judge.json": "0ae462a0661cd552b4c32029e5c14ff6d14f8dcfe1dd08cb90cd7f9da0b9ee87" }, "rubricId": "meeting-followup", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7", "scenarioId": "meeting-followup", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24273 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "meeting-followup", "bytes": 1220, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.09422277777777778 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 6 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24273 }, { "source": "facts", "path": "outcome.durationMs", "value": 169601 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc", "runDir": "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc", "finishedAt": "2026-10-28T08:32:12.407Z", "rawSuccess": true, "rawReason": "craftbook-spreadsheet-model passed 6 deterministic craftbook checks", "durationMs": 159250, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 4, "judgeArtifactCandidates": [ "workspace/spreadsheet-financial-model-workflow-eval/scope.md", "artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 124.76, "nativePrefillTokensPerSec": 1867.89, "inputTokens": 66545, "outputTokens": 7807, "peakRssMb": 18272, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 159250, "derived": { "durationMs": 151560, "meanTokensPerSec": 124.76, "genTokensPerSec": 124.76, "genTokens": 7807, "promptTokensPerSec": 1867.89, "peakRssMb": 18272, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 53463, "output": 2467 } } }, "usage": { "available": true, "totalInputTokens": 66545, "totalOutputTokens": 7807, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 53463, "todayTokensCached": 40429, "todayTokensOut": 2467, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 53463, "totalTokensCached": 40429, "totalTokensOut": 2467, "totalCost": 0, "medianOutputTokensPerSec": 113.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 113.9, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 6559, "cachedInputTokens": 5395, "outputTokens": 311, "cost": 0, "durationMs": 9766, "outputTokensPerSec": 105.11335797506914, "at": "2026-10-28T08:32:06.996Z" }, "lastUpdated": "2026-10-28T08:32:06.996Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:32:07.776Z" } }, "source": "engine-log" }, "process": { "sampleCount": 28, "peakRssMb": 18272, "peakCpuPercent": 264.2, "firstAtMs": 0, "lastAtMs": 150465 }, "gpu": { "available": true, "sampleCount": 28, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 29029, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:29:33.157Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T08:44:26.877Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete task-artifact scope.md, original frozen prompt and exact scenario axes, plus all three seeded source files (craftbook-eval-brief.md, brief.md, records.csv). Source review only; this judgment does not assert execution of the separate HTML model or workflow gates.", "suiteId": "productivity", "scenarioId": "craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc", "artifactPath": "artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 5, "usefulness": 3 }, "justification": "Purpose, drivers, formulas and acceptance checks fit the scope phase. The generic SaaS forecast ignores Boreal Desk's supplied facts and records. Its zero-growth flat-revenue check contradicts 2% churn, and the promised CAC payback output has no formula. The scope needs correction before it can be a reliable build contract.", "artifactSha256": "bc4f69b924094af548ecbb098cb9ad29d162a04f9a204ce6e1c5ffa1ed37c689", "inputBytes": 1500, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4 }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 29029, "totalMb": 31865 }, "evidenceHashes": { "result.json": "e0665c8024558092bf71ead88331aa886a0c86e6772cbd52a0e79c3a08603d21", "score.json": "04dd50266100c1044e355509accc8404f8abb153d83749cbcdd0a5c800428770", "metrics.json": "66b1dfcf8065b33328c0aeaa17c5d4fd7457d0c2c2d8356f1ff4a885709ac78e", "codex-inline-judge.json": "383a47e5f1ec23a80c4fc89946edaf8108e8f6915ddb2884b9b4d1ae49ef8605" }, "rubricId": "craftbook-spreadsheet-model", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc", "scenarioId": "craftbook-spreadsheet-model", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 29292 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-spreadsheet-model", "bytes": 1500, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08847222222222222 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 29292 }, { "source": "facts", "path": "outcome.durationMs", "value": 159250 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9", "runDir": "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9", "finishedAt": "2026-10-28T08:34:40.596Z", "rawSuccess": true, "rawReason": "craftbook-spreadsheet-model passed 6 deterministic craftbook checks", "durationMs": 147432, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 4.5, "judgeArtifactCandidates": [ "artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 125.02, "nativePrefillTokensPerSec": 1883.08, "inputTokens": 61572, "outputTokens": 7517, "peakRssMb": 18936, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 147432, "derived": { "durationMs": 140527, "meanTokensPerSec": 125.02, "genTokensPerSec": 125.02, "genTokens": 7517, "promptTokensPerSec": 1883.08, "peakRssMb": 18936, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43190, "output": 2157 } } }, "usage": { "available": true, "totalInputTokens": 61572, "totalOutputTokens": 7517, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 43190, "todayTokensCached": 31532, "todayTokensOut": 2157, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 43190, "totalTokensCached": 31532, "totalTokensOut": 2157, "totalCost": 0, "medianOutputTokensPerSec": 107.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 107.9, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8646, "cachedInputTokens": 8423, "outputTokens": 80, "cost": 0, "durationMs": 12867, "outputTokensPerSec": 106.50000539240533, "at": "2026-10-28T08:34:25.330Z" }, "lastUpdated": "2026-10-28T08:34:25.330Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:34:36.997Z" } }, "source": "engine-log" }, "process": { "sampleCount": 26, "peakRssMb": 18936, "peakCpuPercent": 275.3, "firstAtMs": 0, "lastAtMs": 139419 }, "gpu": { "available": true, "sampleCount": 26, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 29467, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:32:13.164Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T08:44:26.881Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete task-artifact scope.md, original frozen prompt and exact scenario axes, plus all three seeded source files (craftbook-eval-brief.md, brief.md, records.csv). Source review only; this judgment does not assert execution of the separate HTML model or workflow gates.", "suiteId": "productivity", "scenarioId": "craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9", "artifactPath": "artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 5, "usefulness": 4 }, "justification": "The scope provides a clear question, labeled assumptions, a 12-month axis and implementable growth formulas. It does not adapt to Boreal Desk's supplied facts or records. Target CAC is not used in the calculation contract, and its zero-growth sanity case leaves Actual CAC dividing by zero without a defined result.", "artifactSha256": "29a38967931d8777b83745e9e353c6423be054924a93f7f5a7898a5a79255177", "inputBytes": 1244, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4.5 }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 29467, "totalMb": 31865 }, "evidenceHashes": { "result.json": "d8d91fbe6756c5d2491d99617a17118b7dc4470871066e9181e25dc48284a773", "score.json": "fdbea5409a165b666760ac0573cca62f6c88c822a9c1fd34d751e94787e781fc", "metrics.json": "4a2aa5801021915cdaba3943f67dbf777ab817e816f6d6e7400631abe0e5bf24", "codex-inline-judge.json": "342b6f38a2d7aa5d15ca0180edee1c590833c02b336527cc81ff8c3649e36db5" }, "rubricId": "craftbook-spreadsheet-model", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9", "scenarioId": "craftbook-spreadsheet-model", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34797 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-spreadsheet-model", "bytes": 1244, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08190666666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 18 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34797 }, { "source": "facts", "path": "outcome.durationMs", "value": 147432 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47", "runDir": "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47", "finishedAt": "2026-10-28T08:37:09.362Z", "rawSuccess": true, "rawReason": "craftbook-spreadsheet-model passed 6 deterministic craftbook checks", "durationMs": 147361, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 4.5, "judgeArtifactCandidates": [ "artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 128.8, "nativePrefillTokensPerSec": 1900.47, "inputTokens": 63451, "outputTokens": 6850, "peakRssMb": 18936, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 147361, "derived": { "durationMs": 140292, "meanTokensPerSec": 128.8, "genTokensPerSec": 128.8, "genTokens": 6850, "promptTokensPerSec": 1900.47, "peakRssMb": 18936, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44648, "output": 1259 } } }, "usage": { "available": true, "totalInputTokens": 63451, "totalOutputTokens": 6850, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 44648, "todayTokensCached": 33010, "todayTokensOut": 1259, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 44648, "totalTokensCached": 33010, "totalTokensOut": 1259, "totalCost": 0, "medianOutputTokensPerSec": 120.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 120.1, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8745, "cachedInputTokens": 8538, "outputTokens": 35, "cost": 0, "durationMs": 13784, "outputTokensPerSec": 142.8427385473734, "at": "2026-10-28T08:36:50.069Z" }, "lastUpdated": "2026-10-28T08:36:50.069Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:37:05.579Z" } }, "source": "engine-log" }, "process": { "sampleCount": 26, "peakRssMb": 18936, "peakCpuPercent": 262.9, "firstAtMs": 0, "lastAtMs": 139171 }, "gpu": { "available": true, "sampleCount": 26, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 29274, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:34:42.001Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T08:44:26.882Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete task-artifact scope.md, original frozen prompt and exact scenario axes, plus all three seeded source files (craftbook-eval-brief.md, brief.md, records.csv). Source review only; this judgment does not assert execution of the separate HTML model or workflow gates.", "suiteId": "productivity", "scenarioId": "craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47", "artifactPath": "artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 5, "usefulness": 4 }, "justification": "The phase-shaped scope defines useful margin, LTV, CAC and payback formulas with editable assumptions. It ignores the supplied Boreal Desk context. MRR counts only new customers, the promised 12-month forecast has no customer recurrence, and the sanity scenario gives no expected numeric value. A usable sketch, not a verified model contract.", "artifactSha256": "b173a4e36d44d0529d3bc6d84a42f63c40e30a0ef4a1ab8322180d9c8ef0dfa7", "inputBytes": 1579, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4.5 }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 29274, "totalMb": 31865 }, "evidenceHashes": { "result.json": "d999f2375b0940242f1958462cf6db954bf033cf8e0ad185a8b17e0e92bd49bb", "score.json": "7fe4e1681dc7eb0c9b79d9bb7343294817b8474418db8952cebb0fa4299cbb54", "metrics.json": "144278fe35a57da790cab7cfb6826bdf1de50a67b76c171c234ef97952750c87", "codex-inline-judge.json": "228f4080a943e87057f39292d65b215b1460df418c6471cb889ed950af8235d9" }, "rubricId": "craftbook-spreadsheet-model", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47", "scenarioId": "craftbook-spreadsheet-model", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 28335 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-spreadsheet-model", "bytes": 1579, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08186722222222222 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 18 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 28335 }, { "source": "facts", "path": "outcome.durationMs", "value": 147361 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt", "runDir": "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt", "finishedAt": "2026-10-28T08:39:23.847Z", "rawSuccess": true, "rawReason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 130141, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 127.78, "nativePrefillTokensPerSec": 2014.22, "inputTokens": 139691, "outputTokens": 3287, "peakRssMb": 16799, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 130141, "derived": { "durationMs": 123063, "meanTokensPerSec": 127.78, "genTokensPerSec": 127.78, "genTokens": 3287, "promptTokensPerSec": 2014.22, "peakRssMb": 16799, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 31876, "output": 322 } } }, "usage": { "available": true, "totalInputTokens": 139691, "totalOutputTokens": 3287, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 31876, "todayTokensCached": 30434, "todayTokensOut": 322, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 31876, "totalTokensCached": 30434, "totalTokensOut": 322, "totalCost": 0, "medianOutputTokensPerSec": 96.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 96.7, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11762, "cachedInputTokens": 11275, "outputTokens": 63, "cost": 0, "durationMs": 49715, "outputTokensPerSec": 96.72266180765294, "at": "2026-10-28T08:39:19.321Z" }, "lastUpdated": "2026-10-28T08:39:19.321Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:39:20.036Z" } }, "source": "engine-log" }, "process": { "sampleCount": 23, "peakRssMb": 16799, "peakCpuPercent": 244.4, "firstAtMs": 1, "lastAtMs": 121934 }, "gpu": { "available": true, "sampleCount": 23, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 27112, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:37:13.706Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:43:13.484Z", "suiteId": "productivity", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 23, "peakUsedMb": 27112, "totalMb": 31865 }, "evidenceHashes": { "result.json": "6fe05bc7598331826344510ac044efd7349d2f706784ed8bfdf062920e415a11", "score.json": "90649e2cea5bb26845f16f50232975bbdce99cfe29a8ec5f0bd5c89e69fabe46", "metrics.json": "8968de1957ac27494ba1fac009e788fdb9ccc576f14f53c1d97bbaec7a6025de", "codex-inline-judge.json": "cdea3858684c15f3a0202e4556df0702e0b6e2dc6e24c1a074ab9fcfd107aa9c" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt", "scenarioId": "conflict-synthesis", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24776 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 1508, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07230055555555556 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 18 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24776 }, { "source": "facts", "path": "outcome.durationMs", "value": 130141 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p", "runDir": "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p", "finishedAt": "2026-10-28T08:40:56.476Z", "rawSuccess": true, "rawReason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 91504, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 119.86, "nativePrefillTokensPerSec": 1995.04, "inputTokens": 85992, "outputTokens": 2163, "peakRssMb": 15263, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 91504, "derived": { "durationMs": 84365, "meanTokensPerSec": 119.86, "genTokensPerSec": 119.86, "genTokens": 2163, "promptTokensPerSec": 1995.04, "peakRssMb": 15263, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 25720, "output": 1077 } } }, "usage": { "available": true, "totalInputTokens": 85992, "totalOutputTokens": 2163, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 25720, "todayTokensCached": 17141, "todayTokensOut": 1077, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 25720, "totalTokensCached": 17141, "totalTokensOut": 1077, "totalCost": 0, "medianOutputTokensPerSec": 116.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 116.9, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8132, "cachedInputTokens": 6486, "outputTokens": 528, "cost": 0, "durationMs": 26392, "outputTokensPerSec": 132.69204718901847, "at": "2026-10-28T08:40:50.476Z" }, "lastUpdated": "2026-10-28T08:40:50.476Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:40:52.263Z" } }, "source": "engine-log" }, "process": { "sampleCount": 16, "peakRssMb": 15263, "peakCpuPercent": 244.8, "firstAtMs": 0, "lastAtMs": 83324 }, "gpu": { "available": true, "sampleCount": 16, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 25677, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:39:24.972Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:43:13.495Z", "suiteId": "productivity", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 16, "peakUsedMb": 25677, "totalMb": 31865 }, "evidenceHashes": { "result.json": "6391398f4e2bade976180c045af31e7d40505645de259ad9e560ac1c0884fb4e", "score.json": "e5dbec329de1ee447244e8635a931b34c71c745a259f6709c5f7c2f35dfe0c25", "metrics.json": "f90564fd229793d4bee3854f2d4bdf79d67d7238caed5d7661835a786a4976c1", "codex-inline-judge.json": "047eef9ab3a21c0f3754340f3b9567766b5b9dd0478a8aca25dfad4c5672fa35" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p", "scenarioId": "conflict-synthesis", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25362 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 1876, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.05083555555555556 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 14 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25362 }, { "source": "facts", "path": "outcome.durationMs", "value": 91504 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2", "runDir": "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2", "finishedAt": "2026-10-28T08:41:33.145Z", "rawSuccess": true, "rawReason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 35977, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 122.83, "nativePrefillTokensPerSec": 1974.47, "inputTokens": 11121, "outputTokens": 974, "peakRssMb": 10760, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 35977, "derived": { "durationMs": 28868, "meanTokensPerSec": 122.83, "genTokensPerSec": 122.83, "genTokens": 974, "promptTokensPerSec": 1974.47, "peakRssMb": 10760, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 11118, "output": 121 } } }, "usage": { "available": true, "totalInputTokens": 11121, "totalOutputTokens": 974, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11118, "todayTokensCached": 10666, "todayTokensOut": 121, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11118, "totalTokensCached": 10666, "totalTokensOut": 121, "totalCost": 0, "medianOutputTokensPerSec": 119.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 119.3, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11118, "cachedInputTokens": 10666, "outputTokens": 121, "cost": 0, "durationMs": 20996, "outputTokensPerSec": 119.2828713772798, "at": "2026-10-28T08:41:23.544Z" }, "lastUpdated": "2026-10-28T08:41:23.544Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:41:29.021Z" } }, "source": "engine-log" }, "process": { "sampleCount": 6, "peakRssMb": 10760, "peakCpuPercent": 249.6, "firstAtMs": 1, "lastAtMs": 27829 }, "gpu": { "available": true, "sampleCount": 6, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 21160, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T08:40:57.168Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T08:43:13.503Z", "suiteId": "productivity", "scenarioId": "conflict-synthesis", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 6, "peakUsedMb": 21160, "totalMb": 31865 }, "evidenceHashes": { "result.json": "2bc1d04bcab9cb259d22abbd9392c857d568ef4ec7431a706c39189608954c2e", "score.json": "7123ecf6b76b0d86302728e17ca718bcf5c0be65373436ca1017e7a1f850aae1", "metrics.json": "7fc7ce5f7e890c64403945992a91717526f4046e3f1ef3d170f2c579c4da92fc", "codex-inline-judge.json": "b49c607f18f1f7d81856ffaf72d7e1981eeacefb7fbcf26c2e12f25b112147ca" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2", "scenarioId": "conflict-synthesis", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24786 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 1457, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.019987222222222222 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 24786 }, { "source": "facts", "path": "outcome.durationMs", "value": 35977 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8", "runDir": "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8", "finishedAt": "2026-10-28T08:46:28.977Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on theme-report.md; missing signals stayed unchanged: theme-id, fonts, palette, slide-count, compare-cited, docblocks-provenance. Last failure: infer_theme_from_file never succeeded; call it — describing the step does not count; also: cite the exact theme id custom-brand-reference-pptx returned by infer_theme_from_file", "durationMs": 291894, "rawFixedComposite": 3.8, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 6, "judgeArtifactCandidates": [ "workspace/brand-theme-alignment/theme-report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 115.21, "nativePrefillTokensPerSec": 1898.44, "inputTokens": 122696, "outputTokens": 12918, "peakRssMb": 17370, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 291894, "derived": { "durationMs": 285011, "meanTokensPerSec": 115.21, "genTokensPerSec": 115.21, "genTokens": 12918, "promptTokensPerSec": 1898.44, "peakRssMb": 17370, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 88264, "output": 2939 } } }, "usage": { "available": true, "totalInputTokens": 122696, "totalOutputTokens": 12918, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 88264, "todayTokensCached": 81244, "todayTokensOut": 2939, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 88264, "totalTokensCached": 81244, "totalTokensOut": 2939, "totalCost": 0, "medianOutputTokensPerSec": 111.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 111.4, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12845, "cachedInputTokens": 12708, "outputTokens": 42, "cost": 0, "durationMs": 29567, "outputTokensPerSec": 145.25408837117024, "at": "2026-10-28T08:46:20.831Z" }, "lastUpdated": "2026-10-28T08:46:20.831Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:46:25.078Z" } }, "source": "engine-log" }, "process": { "sampleCount": 52, "peakRssMb": 17370, "peakCpuPercent": 265.6, "firstAtMs": 0, "lastAtMs": 283974 }, "gpu": { "available": true, "sampleCount": 52, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 52, "peakUsedMb": 27924, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8", "status": "excluded", "category": "eval-setup-tool-availability", "reason": "The task and gate require six DocBlocks tools, but the captured model-facing roster contains only 20 workspace file tools. This attempt cannot establish the model's ability to perform the required DocBlocks workflow.", "rawFailureClass": "model", "rawFailureClassRule": "model-default", "evidence": [ "Trial daemon.log:34-35 records the registered bridge and exact model-facing roster; no required DocBlocks tool is present.", "Trial daemon.log:42 and subsequent turns show wire tools=20; result.json ends repair-exhausted on missing DocBlocks provenance.", "evals/src/scenarios/docblocks-theme-roundtrip.ts:370-383 provisions a Copywriter, installs only workspace read/write toolsets, and sends a direct chat without a binary expected deliverable or craftbook context.", "packages/service/src/toolsets/bundled-docblocks.ts:61-68 joins bundled DocBlocks only for a craftbook declaring it or a binary expected deliverable. packages/service/src/chat/manager.ts:15087-15100 applies this policy. The built service contains the same policy." ], "diagnosis": "The scenario's assumption that bundled DocBlocks joins this direct chat no longer matches the runtime policy. This is a setup/tool-contract mismatch, not evidence of missing installed dependencies. No source fix, installation or policy broadening was performed." }, "startedAt": "2026-10-28T08:41:37.083Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-tool-availability", "reason": "The task and gate require six DocBlocks tools, but the captured model-facing roster contains only 20 workspace file tools. This attempt cannot establish the model's ability to perform the required DocBlocks workflow." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T08:52:39.999Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Entire final theme-report.md against the frozen scenario mission, kickoff, context note and exact grounding/candor/usefulness axes. The captured daemon model-facing roster independently confirms the required DocBlocks tools were absent. No PPTX inspection or rendering was performed for this judgment.", "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8", "artifactPath": "workspace/brand-theme-alignment/theme-report.md", "artifactKind": "markdown", "scoreAxes": { "grounding": 8, "candor": 7, "usefulness": 3 }, "justification": "The report labels every theme, slide and comparison value unverified and makes no invented conversion claim. Its missing-tool explanation matches the captured roster. It is candid about the incomplete alignment, but the generic recommendation to align manually gives no owner, steps or verification plan and delivers no usable comparison.", "artifactSha256": "8e78817a526957de33b77122d96cd86b7be099edbea958ae68e126156e62b68b", "inputBytes": 436, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6 }, "systemMemory": { "sampleCount": 52, "peakUsedMb": 27924, "totalMb": 31865 }, "evidenceHashes": { "result.json": "47309e634945e18534b5b9f3a9f3bb3f8a0260ebcef5ae2a95aa0a2d8f50ea79", "score.json": "3f584feff5846b8acefe1635b495e5d87bebc3ca50a10ea45e45296f41a109b0", "metrics.json": "07be510fe1cb62526d7d220791265ff083f94571e9b4cbd27714ffb9a5c8fb95", "codex-inline-judge.json": "ce31b99aa4ddcead557fdef712ad3e9e3f5e53fb256e8f78ac18af334c8b0996" }, "rubricId": "docblocks-theme-roundtrip", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8", "scenarioId": "docblocks-theme-roundtrip", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on theme-report.md; missing signals stayed unchanged: theme-id, fonts, palette, slide-count, compare-cited, docblocks-provenance. Last failure: infer_theme_from_file never succeeded; call it — describing the step does not count; also: cite the exact theme id custom-brand-reference-pptx returned by infer_theme_from_file" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 112008 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 436, "score": 3, "scoreMax": 9, "signals": [ "ordered-sections", "differences-not-whitewashed", "recommendation" ], "failReason": "infer_theme_from_file never succeeded; call it — describing the step does not count; also: cite the exact theme id custom-brand-reference-pptx returned by infer_theme_from_file" } } ] }, "quality": { "score": 3.3, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 33.3% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 112008 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 436, "score": 3, "scoreMax": 9, "signals": [ "ordered-sections", "differences-not-whitewashed", "recommendation" ], "failReason": "infer_theme_from_file never succeeded; call it — describing the step does not count; also: cite the exact theme id custom-brand-reference-pptx returned by infer_theme_from_file" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.16216333333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 17 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 112008 }, { "source": "facts", "path": "outcome.durationMs", "value": 291894 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.8, "band": "framework-gap" } }, { "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao", "runDir": "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao", "finishedAt": "2026-10-28T08:55:40.193Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on theme-report.md; missing signals stayed unchanged: slide-count, compare-cited, docblocks-provenance. Last failure: infer_theme_from_file never succeeded; call it — describing the step does not count; also: save the themed deck to deliverables/quarterly-review.pptx so its slide count can be reported", "durationMs": 550304, "rawFixedComposite": 5.9, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 4.333333333333333, "judgeArtifactCandidates": [ "workspace/brand-theme-alignment/theme-report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 116.62, "nativePrefillTokensPerSec": 1908.36, "inputTokens": 135772, "outputTokens": 13906, "peakRssMb": 18797, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 550304, "derived": { "durationMs": 544388, "meanTokensPerSec": 116.62, "genTokensPerSec": 116.62, "genTokens": 13906, "promptTokensPerSec": 1908.36, "peakRssMb": 18797, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 92087, "output": 1493 } } }, "usage": { "available": true, "totalInputTokens": 135772, "totalOutputTokens": 13906, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 92087, "todayTokensCached": 66473, "todayTokensOut": 1493, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 92087, "totalTokensCached": 66473, "totalTokensOut": 1493, "totalCost": 0, "medianOutputTokensPerSec": 118.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 118.3, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 13944, "cachedInputTokens": 13791, "outputTokens": 37, "cost": 0, "durationMs": 32962, "outputTokensPerSec": 111.34824162568432, "at": "2026-10-28T08:55:36.402Z" }, "lastUpdated": "2026-10-28T08:55:36.402Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T08:55:37.220Z" } }, "source": "engine-log" }, "process": { "sampleCount": 99, "peakRssMb": 18797, "peakCpuPercent": 262.2, "firstAtMs": 0, "lastAtMs": 543282 }, "gpu": { "available": true, "sampleCount": 99, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 99, "peakUsedMb": 29389, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao", "status": "excluded", "category": "eval-setup-tool-availability", "reason": "This repetition independently confirms the same missing six DocBlocks tools in its 20-tool model-facing workspace roster; the required workflow cannot execute through that surface.", "rawFailureClass": "model", "rawFailureClassRule": "model-default", "evidence": [ "This trial's daemon.log:35 records the exact model-facing workspace-only roster, without any required DocBlocks tool.", "This trial's daemon.log:42,103,141,227,261,292,373,445,501,581 retain wire tools=20, including after the harness's direct kick.", "This trial's result.json records repair-exhausted with no successful infer_theme_from_file, saved themed deck or comparison evidence.", "Scenario setup and the source/built bundledDocblocksJoinsSession policy are the same verified inputs as repetition 1; no changes were made between attempts." ], "diagnosis": "Eval setup/tool-contract mismatch. The final report's unsupported high-alignment claim is separately reflected in its qualitative judgment; it does not make the missing runtime prerequisites available. No rerun or source change was made." }, "startedAt": "2026-10-28T08:46:29.889Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-tool-availability", "reason": "This repetition independently confirms the same missing six DocBlocks tools in its 20-tool model-facing workspace roster; the required workflow cannot execute through that surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T08:58:29.142Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Entire final theme-report.md against the frozen scenario mission, kickoff, context note and exact grounding/candor/usefulness axes. Captured daemon roster independently confirms all six required DocBlocks tools were absent. No PPTX inspection or rendering was performed for this judgment.", "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao", "artifactPath": "workspace/brand-theme-alignment/theme-report.md", "artifactKind": "markdown", "scoreAxes": { "grounding": 5, "candor": 4, "usefulness": 4 }, "justification": "The named theme, fonts and colors match the supplied reference values, and unavailable slide/count comparison evidence is acknowledged. However, 'The alignment is high' asserts a result the report cannot verify and softens the incomplete outcome. Consistent chart color is actionable but does not address completing or checking the missing conversion.", "artifactSha256": "ec4ab0be64e6ce7d7f9a1c78d39eccf0e2f639fa6dd467724769c3c4ff48d827", "inputBytes": 442, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4.333333333333333 }, "systemMemory": { "sampleCount": 99, "peakUsedMb": 29389, "totalMb": 31865 }, "evidenceHashes": { "result.json": "4f2b67c779cffc3c2d5f84cd062cb3b15e165ba4d930b865394ba27a9c0af332", "score.json": "8227212aac8705c87dd1e665a4bb826d2950324444c5cdac0836fdfb7cf30eb8", "metrics.json": "950c671b92a1c0f0d204390d8457b8e1e84d8308705aafddc989c548b8dacbf2", "codex-inline-judge.json": "c42cfdb4b8d58486d96485564c84e87a31fa086dc8d76b962b905938407c8d24" }, "rubricId": "docblocks-theme-roundtrip", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao", "scenarioId": "docblocks-theme-roundtrip", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on theme-report.md; missing signals stayed unchanged: slide-count, compare-cited, docblocks-provenance. Last failure: infer_theme_from_file never succeeded; call it — describing the step does not count; also: save the themed deck to deliverables/quarterly-review.pptx so its slide count can be reported" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 106561 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 442, "score": 6, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "differences-not-whitewashed", "recommendation" ], "failReason": "infer_theme_from_file never succeeded; call it — describing the step does not count; also: save the themed deck to deliverables/quarterly-review.pptx so its slide count can be reported" } } ] }, "quality": { "score": 6.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 66.7% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 106561 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 442, "score": 6, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "differences-not-whitewashed", "recommendation" ], "failReason": "infer_theme_from_file never succeeded; call it — describing the step does not count; also: save the themed deck to deliverables/quarterly-review.pptx so its slide count can be reported" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.30572444444444447 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 18 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 106561 }, { "source": "facts", "path": "outcome.durationMs", "value": 550304 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.9, "band": "needs-tuning" } }, { "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3", "runDir": "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3", "finishedAt": "2026-10-28T09:04:12.994Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on theme-report.md; missing signals stayed unchanged: slide-count, compare-cited, docblocks-provenance. Last failure: infer_theme_from_file never succeeded; call it — describing the step does not count; also: save the themed deck to deliverables/quarterly-review.pptx so its slide count can be reported", "durationMs": 512110, "rawFixedComposite": 5.9, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 5.666666666666667, "judgeArtifactCandidates": [ "workspace/brand-theme-alignment/theme-report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 115.1, "nativePrefillTokensPerSec": 1828.43, "inputTokens": 55410, "outputTokens": 13599, "peakRssMb": 17099, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 512110, "derived": { "durationMs": 503736, "meanTokensPerSec": 115.1, "genTokensPerSec": 115.1, "genTokens": 13599, "promptTokensPerSec": 1828.43, "peakRssMb": 17099, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 89780, "output": 6087 } } }, "usage": { "available": true, "totalInputTokens": 55410, "totalOutputTokens": 13599, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 89780, "todayTokensCached": 57289, "todayTokensOut": 6087, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 89780, "totalTokensCached": 57289, "totalTokensOut": 6087, "totalCost": 0, "medianOutputTokensPerSec": 120.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 120.4, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12182, "cachedInputTokens": 11520, "outputTokens": 1085, "cost": 0, "durationMs": 10377, "outputTokensPerSec": 113.38367749916608, "at": "2026-10-28T09:04:06.754Z" }, "lastUpdated": "2026-10-28T09:04:06.754Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:04:07.859Z" } }, "source": "engine-log" }, "process": { "sampleCount": 92, "peakRssMb": 17099, "peakCpuPercent": 268.1, "firstAtMs": 0, "lastAtMs": 502719 }, "gpu": { "available": true, "sampleCount": 92, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 92, "peakUsedMb": 27783, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3", "status": "excluded", "category": "eval-setup-tool-availability", "reason": "The third repetition also exposes only 20 workspace tools and none of the six DocBlocks prerequisites; its required inference/conversion/comparison workflow is unavailable.", "rawFailureClass": "model", "rawFailureClassRule": "model-default", "evidence": [ "This trial's daemon.log:35 independently records the exact workspace-only model-facing roster.", "This trial's daemon.log:42,103,166,264,304,331,366,397,428,472 retain wire tools=20 through every request.", "This trial's result.json records repair-exhausted after four completed attempts, without successful infer_theme_from_file or saved/compared deck evidence.", "Scenario setup and the source/built bundledDocblocksJoinsSession policy remain the same verified inputs as repetitions 1 and 2." ], "diagnosis": "Same independently verified eval setup/tool-contract mismatch. The final Markdown artifact is still judged; original outcome and fixed score remain preserved. The separate PowerPoint craftbook trials do receive DocBlocks and successfully call conversion/preview/save/inspection, so this exclusion does not extend to them." }, "startedAt": "2026-10-28T08:55:40.884Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-tool-availability", "reason": "The third repetition also exposes only 20 workspace tools and none of the six DocBlocks prerequisites; its required inference/conversion/comparison workflow is unavailable." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:12:16.739Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Entire final theme-report.md against the frozen mission, kickoff, context note and exact grounding/candor/usefulness axes. Captured daemon.log:35 independently verifies the required DocBlocks tools are absent. No PPTX inspection or rendering was performed for this judgment.", "suiteId": "productivity", "scenarioId": "docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3", "artifactPath": "workspace/brand-theme-alignment/theme-report.md", "artifactKind": "markdown", "scoreAxes": { "grounding": 7, "candor": 7, "usefulness": 3 }, "justification": "The listed theme values match the reference; slide count and comparison are plainly unverified, and the tool limitation is acknowledged. It makes no claim of successful alignment. Asking for theme values already listed is redundant and does not explain how to complete the conversion or verify the saved deck.", "artifactSha256": "6b6c5267d7dd43a0fc23b124d8276e0b076fb77367f149695619c22e61e69536", "artifactBytes": 383, "contextArtifacts": [], "inputBytes": 383, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.666666666666667 }, "systemMemory": { "sampleCount": 92, "peakUsedMb": 27783, "totalMb": 31865 }, "evidenceHashes": { "result.json": "a916b2157872ed1b3940cd95a53ac8a4a0065d1579578f4a664ba0fdfaa6318e", "score.json": "dc7753cd284814c650b61f998bd7aef18e1bce49e81d6ca7ecc7d7de4edd2721", "metrics.json": "d4618ec89ed75a1a7887e667e5470ef900269edaa9fa51ff27e9d6dd29649eb1", "codex-inline-judge.json": "59626bbd70fa7d10f10d7ef41861442bbdf284fad983db33039afe7647bf4f56" }, "rubricId": "docblocks-theme-roundtrip", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3", "scenarioId": "docblocks-theme-roundtrip", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on theme-report.md; missing signals stayed unchanged: slide-count, compare-cited, docblocks-provenance. Last failure: infer_theme_from_file never succeeded; call it — describing the step does not count; also: save the themed deck to deliverables/quarterly-review.pptx so its slide count can be reported" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 114449 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 383, "score": 6, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "differences-not-whitewashed", "recommendation" ], "failReason": "infer_theme_from_file never succeeded; call it — describing the step does not count; also: save the themed deck to deliverables/quarterly-review.pptx so its slide count can be reported" } } ] }, "quality": { "score": 6.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 66.7% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 114449 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 383, "score": 6, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "differences-not-whitewashed", "recommendation" ], "failReason": "infer_theme_from_file never succeeded; call it — describing the step does not count; also: save the themed deck to deliverables/quarterly-review.pptx so its slide count can be reported" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.28450555555555557 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 12 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 114449 }, { "source": "facts", "path": "outcome.durationMs", "value": 512110 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.9, "band": "needs-tuning" } }, { "suiteId": "productivity", "scenarioId": "craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4", "runDir": "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4", "finishedAt": "2026-10-28T09:06:50.750Z", "rawSuccess": true, "rawReason": "craftbook-powerpoint-deck passed 21 deterministic craftbook checks", "durationMs": 153932, "rawFixedComposite": 9, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.5, "judgeArtifactCandidates": [ "workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 122.23, "nativePrefillTokensPerSec": 1744.69, "inputTokens": 63835, "outputTokens": 5015, "peakRssMb": 21069, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 153932, "derived": { "durationMs": 140000, "meanTokensPerSec": 122.23, "genTokensPerSec": 122.23, "genTokens": 5015, "promptTokensPerSec": 1744.69, "peakRssMb": 21069, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 63775, "output": 1132 } } }, "usage": { "available": true, "totalInputTokens": 63835, "totalOutputTokens": 5015, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 63775, "todayTokensCached": 62178, "todayTokensOut": 1132, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 63775, "totalTokensCached": 62178, "totalTokensOut": 1132, "totalCost": 0, "medianOutputTokensPerSec": 139.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 139.8, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 23171, "cachedInputTokens": 22994, "outputTokens": 33, "cost": 0, "durationMs": 39145, "outputTokensPerSec": 154.51621937440245, "at": "2026-10-28T09:06:39.589Z" }, "lastUpdated": "2026-10-28T09:06:39.589Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:06:40.167Z" } }, "source": "engine-log" }, "process": { "sampleCount": 26, "peakRssMb": 21069, "peakCpuPercent": 257.6, "firstAtMs": 3, "lastAtMs": 138865 }, "gpu": { "available": true, "sampleCount": 26, "peakUtilPercent": 100, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 31455, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:04:16.818Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:12:16.745Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete deck.md, seeded brief, sources.md and outline.md under the combined 20 KiB cap, with the original prompt and exact scenario axes. Captured tool chronology confirms successful brief reads at 09:04:43 and 09:05:06 before outline write at 09:05:12. Markdown source review only; no visual PPTX judgment.", "suiteId": "productivity", "scenarioId": "craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4", "artifactPath": "workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "contextArtifactPaths": [ "workspace/powerpoint-grounding-eval/source/brief.md", "artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "artifacts/powerpoint-grounding-eval/tasks/1/outline.md" ], "artifactKind": "markdown", "scoreAxes": { "source-acquisition": 7, "outline-parity": 8, "grounding": 4, "convertibility": 7 }, "justification": "The brief was read and cited before outlining, and all six outline titles appear once in order as concise H1 slides. Core figures and actions are retained. But the extra '66% improvement in speed' metric is unsupported and imprecise; high-volume categories and improved-verification causation also lack source support.", "artifactSha256": "e357e13fda5b1c583b4b0b0076484e7c3bb696afd9b229c9e50351d89215913d", "artifactBytes": 1368, "contextArtifacts": [ { "path": "workspace/powerpoint-grounding-eval/source/brief.md", "bytes": 381, "sha256": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c" }, { "path": "artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "bytes": 1667, "sha256": "9b80f8b560ed2d143afdac56196e97ca2748334dcdc14f5ddefe3de08ab7ff27" }, { "path": "artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "bytes": 2324, "sha256": "92792dec74e4271fda33adba4fd0afbda07de9ab838f35fa439c612f6ebb1ee5" } ], "inputBytes": 5740, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.5 }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 31455, "totalMb": 31865 }, "evidenceHashes": { "result.json": "76dbb5743bfa26954d84c7179dbe08c1abbc7ac4d2fd050be73f03bdc32aeae3", "score.json": "b65ce5e68e4f95af446977cc6b7513003bda48888a8b73e9aac40103bdb57db8", "metrics.json": "dc6b526b2d82d2bdb9acfcbe25e8a1d760ecd965741423783c046f663f9d1a12", "codex-inline-judge.json": "b9fb8ec3cd565a04ed387e9112e690d42d3054bdd213989d52df89e0dbd9760b" }, "rubricId": "craftbook-powerpoint-deck", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4", "scenarioId": "craftbook-powerpoint-deck", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 35527 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-powerpoint-deck", "bytes": 1663, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07330095238095238 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 29 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 35527 }, { "source": "facts", "path": "outcome.durationMs", "value": 153932 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj", "runDir": "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj", "finishedAt": "2026-10-28T09:09:41.798Z", "rawSuccess": true, "rawReason": "craftbook-powerpoint-deck passed 21 deterministic craftbook checks", "durationMs": 169718, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.75, "judgeArtifactCandidates": [ "workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 121.38, "nativePrefillTokensPerSec": 1740.72, "inputTokens": 68678, "outputTokens": 6617, "peakRssMb": 20635, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10604, "gpuMemoryTotalMb": 12227, "durationMs": 169718, "derived": { "durationMs": 157422, "meanTokensPerSec": 121.38, "genTokensPerSec": 121.38, "genTokens": 6617, "promptTokensPerSec": 1740.72, "peakRssMb": 20635, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44444, "output": 1192 } } }, "usage": { "available": true, "totalInputTokens": 68678, "totalOutputTokens": 6617, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 44444, "todayTokensCached": 42680, "todayTokensOut": 1192, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 44444, "totalTokensCached": 42680, "totalTokensOut": 1192, "totalCost": 0, "medianOutputTokensPerSec": 125.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 125.4, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11947, "cachedInputTokens": 11368, "outputTokens": 39, "cost": 0, "durationMs": 25426, "outputTokensPerSec": 119.3624787189265, "at": "2026-10-28T09:08:47.216Z" }, "lastUpdated": "2026-10-28T09:08:47.216Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:09:33.103Z" } }, "source": "engine-log" }, "process": { "sampleCount": 29, "peakRssMb": 20635, "peakCpuPercent": 255.1, "firstAtMs": 1, "lastAtMs": 156311 }, "gpu": { "available": true, "sampleCount": 29, "peakUtilPercent": 99, "peakMemUsedMb": 10604, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 31266, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:06:52.080Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:12:16.748Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete deck.md, seeded brief, sources.md and outline.md under the combined 20 KiB cap, with the original prompt and exact scenario axes. Captured tool chronology confirms successful brief reads at 09:07:18 and 09:07:53 before outline write at 09:07:59. Markdown source review only; no visual PPTX judgment.", "suiteId": "productivity", "scenarioId": "craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj", "artifactPath": "workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "contextArtifactPaths": [ "workspace/powerpoint-grounding-eval/source/brief.md", "artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "artifacts/powerpoint-grounding-eval/tasks/1/outline.md" ], "artifactKind": "markdown", "scoreAxes": { "source-acquisition": 7, "outline-parity": 8, "grounding": 5, "convertibility": 7 }, "justification": "Source acquisition precedes outlining, and all seven titles map exactly to ordered H1 slides. The supplied metrics and three next actions are accurately cited without extra numbers. Claims of representative product variety and validation across multiple categories are unsupported. Short blocks convert cleanly, though several bullets add generic repetition.", "artifactSha256": "68788bb4c73132cb4f9fc425f84bdffce09d6d6939f203255b3d3a5f93874cdc", "artifactBytes": 1937, "contextArtifacts": [ { "path": "workspace/powerpoint-grounding-eval/source/brief.md", "bytes": 381, "sha256": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c" }, { "path": "artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "bytes": 2143, "sha256": "715034706c8d3fca4e5c23ce7a8b0cb623dc9409d55941dd65c8af99fa01c36e" }, { "path": "artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "bytes": 2462, "sha256": "460a8fb5497d743999c5f1eebf0bfc582c9cf3b1013f85386b7ec048b383d6b2" } ], "inputBytes": 6923, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.75 }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 31266, "totalMb": 31865 }, "evidenceHashes": { "result.json": "67f0477e861c340f1754c614604e597a28b36565e50902bc5ccc9d537565c29b", "score.json": "8b55982b76d8beb56bc295a5baaf2b81b394471e2d6d39823ac6de38b503bcfb", "metrics.json": "11f26fa8175d9f4c2573bb71818c899798e5a6543bfa1ac2d70c3fd0e5e70eb6", "codex-inline-judge.json": "afc923d3d9b6f4b3edebc783e75a974019a97b95aba8ec3eb0136790fd399e4b" }, "rubricId": "craftbook-powerpoint-deck", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj", "scenarioId": "craftbook-powerpoint-deck", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 43664 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-powerpoint-deck", "bytes": 2141, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08081809523809524 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 31 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 43664 }, { "source": "facts", "path": "outcome.durationMs", "value": 169718 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw", "runDir": "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw", "finishedAt": "2026-10-28T09:12:16.169Z", "rawSuccess": true, "rawReason": "craftbook-powerpoint-deck passed 21 deterministic craftbook checks", "durationMs": 153446, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.5, "judgeArtifactCandidates": [ "workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 125.67, "nativePrefillTokensPerSec": 1733.81, "inputTokens": 67092, "outputTokens": 5828, "peakRssMb": 21053, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 153446, "derived": { "durationMs": 141573, "meanTokensPerSec": 125.67, "genTokensPerSec": 125.67, "genTokens": 5828, "promptTokensPerSec": 1733.81, "peakRssMb": 21053, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 42704, "output": 1167 } } }, "usage": { "available": true, "totalInputTokens": 67092, "totalOutputTokens": 5828, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 42704, "todayTokensCached": 41009, "todayTokensOut": 1167, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 42704, "totalTokensCached": 41009, "totalTokensOut": 1167, "totalCost": 0, "medianOutputTokensPerSec": 129.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 129.3, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12105, "cachedInputTokens": 11490, "outputTokens": 39, "cost": 0, "durationMs": 26557, "outputTokensPerSec": 124.98972124002961, "at": "2026-10-28T09:11:23.817Z" }, "lastUpdated": "2026-10-28T09:11:23.817Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:12:07.698Z" } }, "source": "engine-log" }, "process": { "sampleCount": 26, "peakRssMb": 21053, "peakCpuPercent": 261.3, "firstAtMs": 0, "lastAtMs": 140364 }, "gpu": { "available": true, "sampleCount": 26, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 31288, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:09:42.723Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:13:38.149Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete deck.md, seeded brief, sources.md and outline.md under the combined 20 KiB cap, original prompt and exact scenario axes. Captured tool chronology confirms brief reads at 09:10:07 and 09:10:28 before outline write at 09:10:35. Markdown source review only; no visual PPTX judgment.", "suiteId": "productivity", "scenarioId": "craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw", "artifactPath": "workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "contextArtifactPaths": [ "workspace/powerpoint-grounding-eval/source/brief.md", "artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "artifacts/powerpoint-grounding-eval/tasks/1/outline.md" ], "artifactKind": "markdown", "scoreAxes": { "source-acquisition": 7, "outline-parity": 8, "grounding": 4, "convertibility": 7 }, "justification": "The brief was read before outlining, and seven ordered H1 titles exactly match the outline with manageable slide blocks. Original figures and next actions are preserved. The added 66% reduction is absent from the source and imprecisely rounded; claims about highest frustration and customer-satisfaction effects are also unsupported.", "artifactSha256": "bfe3b9f1d6cedd9446b2e17109e6f7ccc4749664e4390b281250fb30374b17b0", "artifactBytes": 2179, "contextArtifacts": [ { "path": "workspace/powerpoint-grounding-eval/source/brief.md", "bytes": 381, "sha256": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c" }, { "path": "artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "bytes": 1656, "sha256": "3ca3131edb37bcc7c8e6a8ddeae185005c903468a997253282cee22436aa4033" }, { "path": "artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "bytes": 2700, "sha256": "aaac4a3317a52ec3d6a3d316173e364cf2b9bfdc74f480bee9d0a4cd7dbc7838" } ], "inputBytes": 6916, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.5 }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 31288, "totalMb": 31865 }, "evidenceHashes": { "result.json": "e3bccf943060d7c30f7de82bd4150626afbe4b29c4048bf20d497f94429a5ad6", "score.json": "e220ba815e33c50e3a705423c1e62554b3c6f6d13494e6ee3f4898b037c0f7b7", "metrics.json": "bd5352d4b4e35afeb0b5a611d2d82ab73d4864fb98047c666b16a3d8bfc5d905", "codex-inline-judge.json": "2cc643e887078e9f1703e0de029f58376a6d3a55fc68ddce98557a61270103e3" }, "rubricId": "craftbook-powerpoint-deck", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw", "scenarioId": "craftbook-powerpoint-deck", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33065 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-powerpoint-deck", "bytes": 1652, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.0730695238095238 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 30 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33065 }, { "source": "facts", "path": "outcome.durationMs", "value": 153446 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "wikipedia-research-brief", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729", "runDir": "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729", "finishedAt": "2026-10-28T09:16:13.405Z", "rawSuccess": true, "rawReason": "research brief is tool-grounded and passes all content gates (word-band, ordered-sections, menabrea-1842, lovelace-1843, notes-a-g, bernoulli, babbage-design, not-completed, source-citations, no-decoy-citation, claim-calibration, no-false-completion, mcp-research-provenance)", "durationMs": 234014, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6, "judgeArtifactCandidates": [ "workspace/analytical-engine-research-brief/research-brief.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 116.89, "nativePrefillTokensPerSec": 1845.7, "inputTokens": 91190, "outputTokens": 16487, "peakRssMb": 18546, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 234014, "derived": { "durationMs": 224413, "meanTokensPerSec": 116.89, "genTokensPerSec": 116.89, "genTokens": 16487, "promptTokensPerSec": 1845.7, "peakRssMb": 18546, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 87825, "output": 4421 } } }, "usage": { "available": true, "totalInputTokens": 91190, "totalOutputTokens": 16487, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 87825, "todayTokensCached": 61895, "todayTokensOut": 4421, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 87825, "totalTokensCached": 61895, "totalTokensOut": 4421, "totalCost": 0, "medianOutputTokensPerSec": 107.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 107.1, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 24747, "cachedInputTokens": 23197, "outputTokens": 1405, "cost": 0, "durationMs": 18347, "outputTokensPerSec": 140.893677356088, "at": "2026-10-28T09:16:03.184Z" }, "lastUpdated": "2026-10-28T09:16:03.184Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:16:07.474Z" } }, "source": "engine-log" }, "process": { "sampleCount": 41, "peakRssMb": 18546, "peakCpuPercent": 267.1, "firstAtMs": 0, "lastAtMs": 223366 }, "gpu": { "available": true, "sampleCount": 41, "peakUtilPercent": 100, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 41, "peakUsedMb": 28600, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:12:19.391Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:21:12.043Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Entire final research brief and the actual local Wikipedia source-card response captured at 09:12:49, under the combined 20 KiB cap, against the original mission/kickoff and exact four scenario axes. The response is preserved verbatim in the recording and parsed in the context snapshot. No live research was used.", "suiteId": "productivity", "scenarioId": "wikipedia-research-brief", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729", "artifactPath": "workspace/analytical-engine-research-brief/research-brief.md", "contextArtifactPaths": [ "judge-context-wikipedia.json" ], "artifactKind": "markdown", "scoreAxes": { "synthesis": 6, "grounding": 5, "citationQuality": 7, "calibration": 6 }, "justification": "Connects the 1842 account, 1843 Notes and Engine design with nearby source IDs and a clear first-programmer caveat. However, symbolic manipulation, modern-software lineage and a century-later realization go beyond the supplied cards without clear qualification. Several sections repeat the same conceptual argument instead of adding supported analysis.", "artifactSha256": "a13fbcb1a918904c4a496f47252d2bc852f1f25a510aa4cad6f34be66354935a", "artifactBytes": 6885, "contextArtifacts": [ { "path": "judge-context-wikipedia.json", "bytes": 2647, "sha256": "19c1f142303c9d66c754ee825f23c09a41a234389565a4c37ab1ec40ee58e7be" } ], "inputBytes": 9532, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6 }, "systemMemory": { "sampleCount": 41, "peakUsedMb": 28600, "totalMb": 31865 }, "evidenceHashes": { "result.json": "c47935fc281e82fb73eb9574d5ec83936d508d98da9613ce98947aca0b45f432", "score.json": "186b92aaf27dfdf55f62bca2daf7542852728b144544ddd661312147e5787792", "metrics.json": "0a7c0e127b89132c56bd03ad4001c278e6c00415aa652a1ecdd936701f4eb5fc", "codex-inline-judge.json": "b0b349081695fac958336a45714e78c03f450a5ebe296b9c73a9d3d5edda9d14" }, "rubricId": "wikipedia-research-brief", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729", "scenarioId": "wikipedia-research-brief", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 35675 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "wikipedia-research", "bytes": 6871, "score": 13, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "claim-calibration", "no-false-completion", "mcp-research-provenance" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.1114352380952381 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 17 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 35675 }, { "source": "facts", "path": "outcome.durationMs", "value": 234014 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "wikipedia-research-brief", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc", "runDir": "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc", "finishedAt": "2026-10-28T09:17:58.180Z", "rawSuccess": true, "rawReason": "research brief is tool-grounded and passes all content gates (word-band, ordered-sections, menabrea-1842, lovelace-1843, notes-a-g, bernoulli, babbage-design, not-completed, source-citations, no-decoy-citation, claim-calibration, no-false-completion, mcp-research-provenance)", "durationMs": 103898, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 3, "judgeArtifactCandidates": [ "workspace/analytical-engine-research-brief/research-brief.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 111.98, "nativePrefillTokensPerSec": 1910.78, "inputTokens": 50712, "outputTokens": 5026, "peakRssMb": 18179, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 103898, "derived": { "durationMs": 95032, "meanTokensPerSec": 111.98, "genTokensPerSec": 111.98, "genTokens": 5026, "promptTokensPerSec": 1910.78, "peakRssMb": 18179, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 50018, "output": 794 } } }, "usage": { "available": true, "totalInputTokens": 50712, "totalOutputTokens": 5026, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 50018, "todayTokensCached": 46719, "todayTokensOut": 794, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 50018, "totalTokensCached": 46719, "totalTokensOut": 794, "totalCost": 0, "medianOutputTokensPerSec": 108, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 108, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14978, "cachedInputTokens": 14004, "outputTokens": 233, "cost": 0, "durationMs": 3152, "outputTokensPerSec": 109.44082812742496, "at": "2026-10-28T09:17:49.823Z" }, "lastUpdated": "2026-10-28T09:17:49.823Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:17:52.606Z" } }, "source": "engine-log" }, "process": { "sampleCount": 18, "peakRssMb": 18179, "peakCpuPercent": 265.5, "firstAtMs": 1, "lastAtMs": 94018 }, "gpu": { "available": true, "sampleCount": 18, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 28228, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:16:14.282Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:21:12.048Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Entire final research brief and actual local Wikipedia source-card response captured at 09:16:41, under the combined 20 KiB cap, against the original mission/kickoff and exact four scenario axes. The response is preserved verbatim in the recording and parsed in the context snapshot. No live research was used.", "suiteId": "productivity", "scenarioId": "wikipedia-research-brief", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc", "artifactPath": "workspace/analytical-engine-research-brief/research-brief.md", "contextArtifactPaths": [ "judge-context-wikipedia.json" ], "artifactKind": "markdown", "scoreAxes": { "synthesis": 4, "grounding": 3, "citationQuality": 2, "calibration": 3 }, "justification": "Covers the core chronology and incomplete Engine, but duplicates the debate section and uses unmapped [2] citations instead of the listed source IDs. Music, conditional logic and peers' reactions lack card support. Saying she was first to write a detailed algorithm reasserts priority after calling the label debated, weakening calibration.", "artifactSha256": "f7fa00c73a115754c796058e702721028bd827e3331d691d02de28cb0dc47b91", "artifactBytes": 5296, "contextArtifacts": [ { "path": "judge-context-wikipedia.json", "bytes": 2647, "sha256": "7291802de1bb87c73b4f0ade731b9a534360e25355572c802e75a8b8e71bcfaf" } ], "inputBytes": 7943, "inputLimitBytes": 20480, "truncated": false, "meanScore": 3 }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 28228, "totalMb": 31865 }, "evidenceHashes": { "result.json": "0e158ba52c63a459bd2ea9e3bb23dfa8ce7f287e9f64c52a5c6de108f1acb14d", "score.json": "d2b58393623a8cee59b36907be7b052a9343eb33b5e165e41120d392d507d2a8", "metrics.json": "52302f76a13e2783674c5b530220d8364a24de75fead98edcfc0465997fea748", "codex-inline-judge.json": "5d38d3bdee9628a5b681fc6141d0ae0744454ac6386dc0e7be17bad870d1a8cd" }, "rubricId": "wikipedia-research-brief", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc", "scenarioId": "wikipedia-research-brief", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33013 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "wikipedia-research", "bytes": 5276, "score": 13, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "claim-calibration", "no-false-completion", "mcp-research-provenance" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.04947523809523809 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 11 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33013 }, { "source": "facts", "path": "outcome.durationMs", "value": 103898 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "wikipedia-research-brief", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw", "runDir": "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw", "finishedAt": "2026-10-28T09:21:05.750Z", "rawSuccess": true, "rawReason": "research brief is tool-grounded and passes all content gates (word-band, ordered-sections, menabrea-1842, lovelace-1843, notes-a-g, bernoulli, babbage-design, not-completed, source-citations, no-decoy-citation, claim-calibration, no-false-completion, mcp-research-provenance)", "durationMs": 186311, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 3.75, "judgeArtifactCandidates": [ "workspace/analytical-engine-research-brief/research-brief.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 106.88, "nativePrefillTokensPerSec": 1924.76, "inputTokens": 99620, "outputTokens": 10146, "peakRssMb": 17631, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 186311, "derived": { "durationMs": 179048, "meanTokensPerSec": 106.88, "genTokensPerSec": 106.88, "genTokens": 10146, "promptTokensPerSec": 1924.76, "peakRssMb": 17631, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 75441, "output": 1050 } } }, "usage": { "available": true, "totalInputTokens": 99620, "totalOutputTokens": 10146, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 75441, "todayTokensCached": 51423, "todayTokensOut": 1050, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 75441, "totalTokensCached": 51423, "totalTokensOut": 1050, "totalCost": 0, "medianOutputTokensPerSec": 97.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 97.6, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 20068, "outputTokens": 475, "cost": 0, "durationMs": 15172, "outputTokensPerSec": 112.36863359978608, "at": "2026-10-28T09:20:59.926Z" }, "lastUpdated": "2026-10-28T09:20:59.926Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:21:01.896Z" } }, "source": "engine-log" }, "process": { "sampleCount": 33, "peakRssMb": 17631, "peakCpuPercent": 264, "firstAtMs": 0, "lastAtMs": 178027 }, "gpu": { "available": true, "sampleCount": 33, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 27809, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:17:59.439Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:22:33.475Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Entire final research brief and actual local Wikipedia source-card response captured at 09:18:30, under the combined 20 KiB cap, against the original mission/kickoff and exact four scenario axes. The parsed context snapshot retains the recording hash. No live research was used.", "suiteId": "productivity", "scenarioId": "wikipedia-research-brief", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw", "artifactPath": "workspace/analytical-engine-research-brief/research-brief.md", "contextArtifactPaths": [ "judge-context-wikipedia.json" ], "artifactKind": "markdown", "scoreAxes": { "synthesis": 6, "grounding": 4, "citationQuality": 2, "calibration": 3 }, "justification": "Links the memoir, architecture and Note G into a coherent account, but [2] citations have no mapping to the listed source IDs. Claims about symbolic computation and later influence exceed the cards. Despite a debated-title caveat, it flatly calls her work the first complex algorithm for a machine, reintroducing the disputed priority claim.", "artifactSha256": "28b458ca370846c4962d1bb9002bdf86d493bbdf04244d774e0ec299a0849258", "artifactBytes": 6555, "contextArtifacts": [ { "path": "judge-context-wikipedia.json", "bytes": 2647, "sha256": "da2e630c240cb8bef80af1ffa2e8f0e84415825816b0df354aec4ed5ddc63da2" } ], "inputBytes": 9202, "inputLimitBytes": 20480, "truncated": false, "meanScore": 3.75 }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 27809, "totalMb": 31865 }, "evidenceHashes": { "result.json": "f1a46971cda5f0f75f2c5562af3ef31dc0a78f0fdeb76fd18de9bd7a6b44f0f5", "score.json": "1393a23a2c4d2224ad77e12579941b28b68dd3030177d31441f54e1fdea85556", "metrics.json": "ee4fd16b8aeabcc011a645b9e2c71d00ed45b43a0588baad08cac267c7c477b8", "codex-inline-judge.json": "276431016f37484bd08934ba2802fe10485fe25b3ab381f55d8121aebe1293fa" }, "rubricId": "wikipedia-research-brief", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw", "scenarioId": "wikipedia-research-brief", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 37097 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "wikipedia-research", "bytes": 6535, "score": 13, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "claim-calibration", "no-false-completion", "mcp-research-provenance" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.0887195238095238 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 13 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 37097 }, { "source": "facts", "path": "outcome.durationMs", "value": 186311 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-week-plan", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud", "runDir": "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud", "finishedAt": "2026-10-28T09:29:25.559Z", "rawSuccess": true, "rawReason": "craftbook-week-plan passed 12 deterministic craftbook checks", "durationMs": 495743, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 1.25, "judgeArtifactCandidates": [ "workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 121.16, "nativePrefillTokensPerSec": 1817.22, "inputTokens": 222499, "outputTokens": 25034, "peakRssMb": 21590, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 495743, "derived": { "durationMs": 483724, "meanTokensPerSec": 121.16, "genTokensPerSec": 121.16, "genTokens": 25034, "promptTokensPerSec": 1817.22, "peakRssMb": 21590, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 229593, "output": 8570 } } }, "usage": { "available": true, "totalInputTokens": 222499, "totalOutputTokens": 25034, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 19, "todayTokensIn": 229593, "todayTokensCached": 197238, "todayTokensOut": 8570, "todayCost": 0, "totalTurns": 19, "totalTokensIn": 229593, "totalTokensCached": 197238, "totalTokensOut": 8570, "totalCost": 0, "medianOutputTokensPerSec": 128.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 128.9, "turns": 19 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7078, "cachedInputTokens": 6809, "outputTokens": 136, "cost": 0, "durationMs": 5395, "outputTokensPerSec": 110.5721741438028, "at": "2026-10-28T09:29:10.432Z" }, "lastUpdated": "2026-10-28T09:29:10.432Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:29:16.634Z" } }, "source": "engine-log" }, "process": { "sampleCount": 87, "peakRssMb": 21590, "peakCpuPercent": 265.9, "firstAtMs": 1, "lastAtMs": 482645 }, "gpu": { "available": true, "sampleCount": 87, "peakUtilPercent": 100, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 87, "peakUsedMb": 31460, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:21:09.816Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:40:23.723Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete required workspace report, all three seeded follow-ups and actual eight-event calendar response from the successful 09:29:15 probe, under the combined 20 KiB cap. Compared against the original prompt and exact coverage/prioritization/conflict-handling/realism axes. Calendar response is copied from the recording with its SHA256; no new calendar call or edits were made.", "suiteId": "productivity", "scenarioId": "craftbook-week-plan", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud", "artifactPath": "workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "contextArtifactPaths": [ "workspace/calendar-desk-week-plan-eval/followups.json", "judge-context-calendar.json" ], "artifactKind": "markdown", "scoreAxes": { "coverage": 1, "prioritization": 2, "conflict-handling": 1, "realism": 1 }, "justification": "Plans an October week instead of July 13 and replaces most of the eight real events with generic meetings. It moves the vendor sync to Wednesday and misses the actual budget-review/Priya overlap. Follow-up names survive, but due dates and real gaps do not; the heavy-day claim and unspecified Priya reschedule are unsupported.", "artifactSha256": "ee46ca6e4e299e27aad357b140bcfaec06fc8c3b3d39e30e0d669c26669e1098", "artifactBytes": 2019, "contextArtifacts": [ { "path": "workspace/calendar-desk-week-plan-eval/followups.json", "bytes": 402, "sha256": "9738e165a80063b91a9a70de118e9b94f4d8d50d662ad1d629c9d01987421757" }, { "path": "judge-context-calendar.json", "bytes": 3015, "sha256": "d44deef37ba4b21e803349992506ec11351be18eb9ed9d2238e1b8ba21c95eec" } ], "inputBytes": 5436, "inputLimitBytes": 20480, "truncated": false, "meanScore": 1.25 }, "systemMemory": { "sampleCount": 87, "peakUsedMb": 31460, "totalMb": 31865 }, "evidenceHashes": { "result.json": "ee24f5b9d2f2049d3a511290328a61c30b48d7c491baaccaef7adc4581211b09", "score.json": "a1d28521ce681869173f9dc752ea0d0b382ff5c405fd996cdcfb44d7a9c3eb96", "metrics.json": "7bbfb6078e032ba74aa96dac76552cbfd964548a853a306230ce32cd8c4b8192", "codex-inline-judge.json": "c361ad96d22d4e0b956544551e86667a52a1ef449bcd772c8c2b23db428b9f92" }, "rubricId": "craftbook-week-plan", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud", "scenarioId": "craftbook-week-plan", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 38846 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-week-plan", "bytes": 1977, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.18360851851851853 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 75 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 38846 }, { "source": "facts", "path": "outcome.durationMs", "value": 495743 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-week-plan", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk", "runDir": "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk", "finishedAt": "2026-10-28T09:38:04.046Z", "rawSuccess": true, "rawReason": "craftbook-week-plan passed 12 deterministic craftbook checks", "durationMs": 517363, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 1.25, "judgeArtifactCandidates": [ "workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 126.33, "nativePrefillTokensPerSec": 1734.55, "inputTokens": 195069, "outputTokens": 28543, "peakRssMb": 21528, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 517363, "derived": { "durationMs": 506547, "meanTokensPerSec": 126.33, "genTokensPerSec": 126.33, "genTokens": 28543, "promptTokensPerSec": 1734.55, "peakRssMb": 21528, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 177267, "output": 6194 } } }, "usage": { "available": true, "totalInputTokens": 195069, "totalOutputTokens": 28543, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 16, "todayTokensIn": 177267, "todayTokensCached": 155579, "todayTokensOut": 6194, "todayCost": 0, "totalTurns": 16, "totalTokensIn": 177267, "totalTokensCached": 155579, "totalTokensOut": 6194, "totalCost": 0, "medianOutputTokensPerSec": 122.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 122.9, "turns": 16 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8372, "cachedInputTokens": 7649, "outputTokens": 546, "cost": 0, "durationMs": 10430, "outputTokensPerSec": 141.6348876094682, "at": "2026-10-28T09:37:55.704Z" }, "lastUpdated": "2026-10-28T09:37:55.704Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:37:56.762Z" } }, "source": "engine-log" }, "process": { "sampleCount": 91, "peakRssMb": 21528, "peakCpuPercent": 270.7, "firstAtMs": 1, "lastAtMs": 505543 }, "gpu": { "available": true, "sampleCount": 91, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 91, "peakUsedMb": 31510, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:29:26.683Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:40:23.728Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete required workspace report, all three seeded follow-ups and actual eight-event calendar response from the successful 09:37:50 probe, under the combined 20 KiB cap. Compared against the original prompt and exact coverage/prioritization/conflict-handling/realism axes. Calendar response is copied from the recording with its SHA256; no new calendar call or edits were made.", "suiteId": "productivity", "scenarioId": "craftbook-week-plan", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk", "artifactPath": "workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "contextArtifactPaths": [ "workspace/calendar-desk-week-plan-eval/followups.json", "judge-context-calendar.json" ], "artifactKind": "markdown", "scoreAxes": { "coverage": 1, "prioritization": 2, "conflict-handling": 1, "realism": 1 }, "justification": "Uses October dates for a July calendar, invents Project Sync/Data Review/Stakeholder Review, and omits most real events. Vendor sync has the wrong time; merely mentioning Priya does not resolve her budget-review overlap. Follow-ups are named, but focus blocks lack times and heavy-day claims rest on invented meetings.", "artifactSha256": "222a0709e8480ed0744569df7956b7359ad52a94c11a01c6103c480133996f25", "artifactBytes": 1921, "contextArtifacts": [ { "path": "workspace/calendar-desk-week-plan-eval/followups.json", "bytes": 402, "sha256": "9738e165a80063b91a9a70de118e9b94f4d8d50d662ad1d629c9d01987421757" }, { "path": "judge-context-calendar.json", "bytes": 3015, "sha256": "959d0de77c9ea6c1e7c94eeec235ce7ee7ea7d675d16371ee4002ad31fcc28cd" } ], "inputBytes": 5338, "inputLimitBytes": 20480, "truncated": false, "meanScore": 1.25 }, "systemMemory": { "sampleCount": 91, "peakUsedMb": 31510, "totalMb": 31865 }, "evidenceHashes": { "result.json": "b3dfa118a6db5acd9aea8b4e8138d31c24b87eeadd9805dacf0bad2be4466382", "score.json": "bec9a73d2ba6285608861b34df80405e51cf447a4b086cb1cefa940c381f219c", "metrics.json": "bc2bdc792c35e6d57f71da405907b0ee0ea02cc4ae38eb83f81531f84550626d", "codex-inline-judge.json": "2e87a79443316279154a5dc4f4d00bdea222ccb15da8bb7266c33895563d8479" }, "rubricId": "craftbook-week-plan", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk", "scenarioId": "craftbook-week-plan", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 37895 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-week-plan", "bytes": 1921, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.19161592592592594 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 99 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 37895 }, { "source": "facts", "path": "outcome.durationMs", "value": 517363 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-week-plan", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng", "runDir": "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng", "finishedAt": "2026-10-28T09:44:17.151Z", "rawSuccess": true, "rawReason": "craftbook-week-plan passed 12 deterministic craftbook checks", "durationMs": 372414, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 1.25, "judgeArtifactCandidates": [ "workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 117.06, "nativePrefillTokensPerSec": 1823.39, "inputTokens": 141069, "outputTokens": 13263, "peakRssMb": 21518, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 372414, "derived": { "durationMs": 363302, "meanTokensPerSec": 117.06, "genTokensPerSec": 117.06, "genTokens": 13263, "promptTokensPerSec": 1823.39, "peakRssMb": 21518, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 147644, "output": 4272 } } }, "usage": { "available": true, "totalInputTokens": 141069, "totalOutputTokens": 13263, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 147644, "todayTokensCached": 128972, "todayTokensOut": 4272, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 147644, "totalTokensCached": 128972, "totalTokensOut": 4272, "totalCost": 0, "medianOutputTokensPerSec": 120.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 120.9, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7049, "cachedInputTokens": 6209, "outputTokens": 1158, "cost": 0, "durationMs": 15226, "outputTokensPerSec": 104.22149292199143, "at": "2026-10-28T09:43:57.181Z" }, "lastUpdated": "2026-10-28T09:43:57.181Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:44:11.867Z" } }, "source": "engine-log" }, "process": { "sampleCount": 66, "peakRssMb": 21518, "peakCpuPercent": 262.7, "firstAtMs": 0, "lastAtMs": 362235 }, "gpu": { "available": true, "sampleCount": 66, "peakUtilPercent": 100, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 66, "peakUsedMb": 31033, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T09:38:04.737Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:52:23.533Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete required workspace report, all seeded follow-ups and actual eight-event calendar response from the successful 09:44:06 probe, under the combined 20 KiB cap. Compared with the original prompt and exact coverage/prioritization/conflict-handling/realism axes. The context snapshot retains the recording hash; no new service call or calendar edit was made.", "suiteId": "productivity", "scenarioId": "craftbook-week-plan", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng", "artifactPath": "workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "contextArtifactPaths": [ "workspace/calendar-desk-week-plan-eval/followups.json", "judge-context-calendar.json" ], "artifactKind": "markdown", "scoreAxes": { "coverage": 1, "prioritization": 2, "conflict-handling": 1, "realism": 1 }, "justification": "Plans October 26 instead of the returned July week, replaces real events with a kickoff/client-call conflict, and gives the vendor sync the wrong time. Priya is mentioned without resolving her actual budget-review overlap. Follow-up IDs survive, but research/testing blocks are invented and clash with omitted calendar obligations.", "artifactSha256": "d825b8892b9a924e38e477f693a56000185fbbee515f210d1101e47de45749e7", "artifactBytes": 2114, "contextArtifacts": [ { "path": "workspace/calendar-desk-week-plan-eval/followups.json", "bytes": 402, "sha256": "9738e165a80063b91a9a70de118e9b94f4d8d50d662ad1d629c9d01987421757" }, { "path": "judge-context-calendar.json", "bytes": 3015, "sha256": "9ece25149614990ecbdf4324a3b9c79d47b0ac7d76264b32ffbff946059edf72" } ], "inputBytes": 5531, "inputLimitBytes": 20480, "truncated": false, "meanScore": 1.25 }, "systemMemory": { "sampleCount": 66, "peakUsedMb": 31033, "totalMb": 31865 }, "evidenceHashes": { "result.json": "c2906abe39e9de24fcdf6c1d55d70089863bc25240b6906bf3e250550a295593", "score.json": "ebc3e039b548d40468a7adc68428009f8b8aea0c198c701ff6b54db26e926480", "metrics.json": "1fe5863e5469b5a5f66caf4d0bd29e01c7c042585e8c287eb6b9f2c2f69371ec", "codex-inline-judge.json": "cd6a2d868b629907d5da4b7977533adaf959eeea19e71e4b7781e79e97cf870e" }, "rubricId": "craftbook-week-plan", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng", "scenarioId": "craftbook-week-plan", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 40950 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-week-plan", "bytes": 2112, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.13793111111111112 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 43 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 40950 }, { "source": "facts", "path": "outcome.durationMs", "value": 372414 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s", "runDir": "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s", "finishedAt": "2026-10-28T09:49:12.689Z", "rawSuccess": true, "rawReason": "craftbook-ab-test-readout passed 19 deterministic craftbook checks", "durationMs": 291565, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 0.25, "judgeArtifactCandidates": [ "workspace/a-b-test-readout-eval/tasks/eval/readout.md", "artifacts/a-b-test-readout-eval/tasks/eval/readout.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 121.64, "nativePrefillTokensPerSec": 1928.81, "inputTokens": 80330, "outputTokens": 9765, "peakRssMb": 19178, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 291565, "derived": { "durationMs": 283016, "meanTokensPerSec": 121.64, "genTokensPerSec": 121.64, "genTokens": 9765, "promptTokensPerSec": 1928.81, "peakRssMb": 19178, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 75555, "output": 2661 } } }, "usage": { "available": true, "totalInputTokens": 80330, "totalOutputTokens": 9765, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 75555, "todayTokensCached": 73467, "todayTokensOut": 2661, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 75555, "totalTokensCached": 73467, "totalTokensOut": 2661, "totalCost": 0, "medianOutputTokensPerSec": 120.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 120.3, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10844, "cachedInputTokens": 10758, "outputTokens": 43, "cost": 0, "durationMs": 11877, "outputTokensPerSec": 97.19747286570549, "at": "2026-10-28T09:49:03.341Z" }, "lastUpdated": "2026-10-28T09:49:03.341Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T09:49:07.454Z" } }, "source": "engine-log" }, "process": { "sampleCount": 52, "peakRssMb": 19178, "peakCpuPercent": 269.8, "firstAtMs": 2, "lastAtMs": 281919 }, "gpu": { "available": true, "sampleCount": 52, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 52, "peakUsedMb": 28925, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "judgmentAggregateEligibility": "excluded-rubric-mismatch", "judgmentRubricLimitation": "The frozen safety/mapping/signals/containment descriptions concern fake endpoints, dry-run credentials, retry/abort signals and mock-service containment. The requested artifact is an experiment analysis and contains no simulator plan. Low literal scores are not evidence of poor A/B analysis or unsafe behavior.", "startedAt": "2026-10-28T09:44:21.124Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T09:54:14.415Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete required workspace readout, source brief, input CSV and produced stats.json, under the combined 20 KiB cap. Exact frozen axes applied literally; their simulator-plan requirements are unrelated to this A/B readout. This is an actual artifact review, but these numeric axis scores are excluded from qualitative aggregates because of the rubric mismatch. Deterministic capability and performance remain eligible.", "suiteId": "productivity", "scenarioId": "craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s", "artifactPath": "workspace/a-b-test-readout-eval/tasks/eval/readout.md", "contextArtifactPaths": [ "workspace/a-b-test-readout-eval/source/experiment-brief.md", "workspace/a-b-test-readout-eval/data/checkout-test.csv", "workspace/a-b-test-readout-eval/data/stats.json" ], "artifactKind": "markdown", "aggregateEligibility": "excluded-rubric-mismatch", "rubricLimitation": "The frozen safety/mapping/signals/containment descriptions concern fake endpoints, dry-run credentials, retry/abort signals and mock-service containment. The requested artifact is an experiment analysis and contains no simulator plan. Low literal scores are not evidence of poor A/B analysis or unsafe behavior.", "taskArtifactReview": "The readout correctly leads with Do not ship, retains the observed rates, +1.25 pp conversion lift, +10.42% relative lift, p=0.0078 and +0.23 pp refund delta, and applies the 0.20 pp guardrail despite significant primary improvement. It calls the refund increase a significant result without a separate refund significance test; distinguish the preregistered threshold breach from statistical significance. No replacement rubric or new numeric task-quality score was invented.", "scoreAxes": { "safety": 0, "mapping": 0, "signals": 1, "containment": 0 }, "justification": "Rubric mismatch: this readout supplies no dry-run boundary, fake endpoint mapping or mock containment plan. It defines ship/no-ship outcomes but no simulator retry/abort signals. These literal scores are excluded from qualitative aggregation; the artifact correctly applies the A/B guardrail and is reviewed separately in taskArtifactReview.", "artifactSha256": "a2e9eca6ccecbba451f318b33d943904cb0b21038aa3811a5312e33800921045", "artifactBytes": 2271, "contextArtifacts": [ { "path": "workspace/a-b-test-readout-eval/source/experiment-brief.md", "bytes": 504, "sha256": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed" }, { "path": "workspace/a-b-test-readout-eval/data/checkout-test.csv", "bytes": 95, "sha256": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35" }, { "path": "workspace/a-b-test-readout-eval/data/stats.json", "bytes": 438, "sha256": "f2d50a862f1ff5922b62b62ac4883aa4e19ac25000d3df6263c5597a84de8113" } ], "inputBytes": 3308, "inputLimitBytes": 20480, "truncated": false, "meanScore": 0.25 }, "systemMemory": { "sampleCount": 52, "peakUsedMb": 28925, "totalMb": 31865 }, "evidenceHashes": { "result.json": "03983b71ff59693d3d8f80bcdf766c6db426e513b5e9047b3ee09ad5c5976909", "score.json": "5cfc490b95566c8ee57e415cb338d3e5ff66b0f190810c73af6c2eef9def53a5", "metrics.json": "730241055ac2bd6f590b64dff862ecc67ae6f84bf82c541fc1e2a957cb57c41e", "codex-inline-judge.json": "54515d03a57dff96e2dd3e0b70ddccfdd3adc04ee3bf0b701541eea0bc33b943" }, "rubricId": "craftbook-ab-test-readout", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s", "scenarioId": "craftbook-ab-test-readout", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34579 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-ab-test-readout", "bytes": 438, "score": 19, "scoreMax": 19, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.10798703703703703 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34579 }, { "source": "facts", "path": "outcome.durationMs", "value": 291565 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u", "runDir": "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u", "finishedAt": "2026-10-28T11:08:51.297Z", "rawSuccess": false, "rawReason": "chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 35c35a4e6b311a39) (re-engage nudge was sent and ignored)", "durationMs": 4778004, "rawFixedComposite": 1.2, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 0.25, "judgeArtifactCandidates": [ "artifacts/a-b-test-readout-eval/tasks/2/readout.md", "artifacts/a-b-test-readout-eval/tasks/3/readout.md", "artifacts/a-b-test-readout-eval/tasks/eval/readout.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 116.94, "nativePrefillTokensPerSec": 1798.69, "inputTokens": 807328, "outputTokens": 135672, "peakRssMb": 21316, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 4778004, "derived": { "durationMs": 4773324, "meanTokensPerSec": 116.94, "genTokensPerSec": 116.94, "genTokens": 135672, "promptTokensPerSec": 1798.69, "peakRssMb": 21316, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 1011026, "output": 36517 } } }, "usage": { "available": true, "totalInputTokens": 807328, "totalOutputTokens": 135672, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 88, "todayTokensIn": 1011026, "todayTokensCached": 927510, "todayTokensOut": 36517, "todayCost": 0, "totalTurns": 88, "totalTokensIn": 1011026, "totalTokensCached": 927510, "totalTokensOut": 36517, "totalCost": 0, "medianOutputTokensPerSec": 132.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 132.6, "turns": 88 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9867, "cachedInputTokens": 9370, "outputTokens": 352, "cost": 0, "durationMs": 9946, "outputTokensPerSec": 122.14584143869008, "at": "2026-10-28T11:03:42.785Z" }, "lastUpdated": "2026-10-28T11:03:42.785Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:08:50.002Z" } }, "source": "engine-log" }, "process": { "sampleCount": 864, "peakRssMb": 21316, "peakCpuPercent": 271.2, "firstAtMs": 0, "lastAtMs": 4772280 }, "gpu": { "available": true, "sampleCount": 864, "peakUtilPercent": 100, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 864, "peakUsedMb": 31216, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "chat-stalled", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] a-b-test-readout-eval/1 paused for help (gate_plateau) at step \"write\"", "nativeEngineIncidents": null, "judgmentAggregateEligibility": "excluded-rubric-mismatch", "judgmentRubricLimitation": "The frozen safety/mapping/signals/containment descriptions concern fake endpoints, dry-run credentials, retry/abort signals and mock-service containment. These experiment readouts contain no simulator plan. Low literal scores do not measure A/B quality or prove unsafe behavior.", "startedAt": "2026-10-28T09:49:13.293Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T11:11:51.257Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "All three complete saved readout.md artifacts, original brief, CSV and final stats.json read under the combined 20 KiB cap. The closest requested-path artifact is the primary; no required workspace readout exists. Exact frozen simulator axes were applied literally and excluded from qualitative aggregates as unrelated to A/B analysis. Deterministic failure is preserved; artifact presence is not mislabeled noArtifact.", "suiteId": "productivity", "scenarioId": "craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u", "artifactPath": "artifacts/a-b-test-readout-eval/tasks/eval/readout.md", "contextArtifactPaths": [ "artifacts/a-b-test-readout-eval/tasks/2/readout.md", "artifacts/a-b-test-readout-eval/tasks/3/readout.md", "workspace/a-b-test-readout-eval/source/experiment-brief.md", "workspace/a-b-test-readout-eval/data/checkout-test.csv", "workspace/a-b-test-readout-eval/data/stats.json" ], "artifactKind": "markdown", "aggregateEligibility": "excluded-rubric-mismatch", "rubricLimitation": "The frozen safety/mapping/signals/containment descriptions concern fake endpoints, dry-run credentials, retry/abort signals and mock-service containment. These experiment readouts contain no simulator plan. Low literal scores do not measure A/B quality or prove unsafe behavior.", "taskArtifactReview": "The 1671-byte tasks/eval artifact correctly recommends Do Not Ship, reports +1.25 pp conversion lift and p=0.0078, and applies the +0.23 pp refund breach against 0.20 pp. It is below the 2000-byte gate and exists only in artifacts. Later task 2/3 copies also reject shipping but add unsupported power claims and behavioral explanations; task 3 adds a relative-lift interval absent from final stats and mixes percent/percentage-point labels. Final stats retain correct underlying rates but violate the required JSON schema and units. No new numeric task-quality rubric was invented.", "scoreAxes": { "safety": 0, "mapping": 0, "signals": 1, "containment": 0 }, "justification": "Rubric mismatch: these readouts have no dry-run boundary, fake endpoint mapping or containment plan. Ship/no-ship conditions provide minimal outcome signals but no simulator retry/abort policy. Exclude these literal scores from qualitative aggregates. Actual A/B content and the missing workspace deliverable are reviewed separately.", "artifactSha256": "aaf6fbfb4dbf13bcda54e2898bcc7af4315be8edf7f6df651200a2771a460d4c", "artifactBytes": 1671, "contextArtifacts": [ { "path": "artifacts/a-b-test-readout-eval/tasks/2/readout.md", "bytes": 2019, "sha256": "c748302a9ee4b6bdec5342c0c10667f9769e4657ee591597a0c40ba15f7cf7f5" }, { "path": "artifacts/a-b-test-readout-eval/tasks/3/readout.md", "bytes": 2833, "sha256": "72ed4f252e6ebf654072cb07b6f695474f0d51fbfea5c561c17247feb166dd5e" }, { "path": "workspace/a-b-test-readout-eval/source/experiment-brief.md", "bytes": 504, "sha256": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed" }, { "path": "workspace/a-b-test-readout-eval/data/checkout-test.csv", "bytes": 95, "sha256": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35" }, { "path": "workspace/a-b-test-readout-eval/data/stats.json", "bytes": 703, "sha256": "0d470a2734ff27c0788ab2cfc159b89d8770d198cb6b1c119214d19773bc0033" } ], "inputBytes": 7825, "inputLimitBytes": 20480, "truncated": false, "meanScore": 0.25 }, "systemMemory": { "sampleCount": 864, "peakUsedMb": 31216, "totalMb": 31865 }, "evidenceHashes": { "result.json": "d56cb760a4f48ac96be1cd6812562b6c2523c21feed26b27c620bf263a064fb7", "score.json": "9d51b0e07a945036b62ecc27118df82be33c3aa5dfe8532cbd2f47aa89c73787", "metrics.json": "18b79d8c13d32dea4e92f4da2d11f11e9835f982d66c81d58dbaf2403689540e", "codex-inline-judge.json": "53bfaf5154013f6fa8a371d355fcbeb077914b4bad23bec79cb516cb43f0a87e" }, "rubricId": "craftbook-ab-test-readout", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u", "scenarioId": "craftbook-ab-test-readout", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "chat-stalled" }, { "source": "facts", "path": "outcome.reason", "value": "chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 35c35a4e6b311a39) (re-engage nudge was sent and ignored)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33745 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-ab-test-readout", "bytes": 703, "score": 3, "scoreMax": 19, "signals": [], "failReason": "data/stats.json control.rate should equal 0.12 but was undefined: control conversion rate is 1200/10000" } } ] }, "quality": { "score": 1.6, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 15.8% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33745 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-ab-test-readout", "bytes": 703, "score": 3, "scoreMax": 19, "signals": [], "failReason": "data/stats.json control.rate should equal 0.12 but was undefined: control conversion rate is 1200/10000" } } ] }, "efficiency": { "score": 0, "ruleId": "E_BOTH_SEVERE", "summary": "The trial exhausted its budget while making at least thirty tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 325 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33745 }, { "source": "facts", "path": "outcome.durationMs", "value": 4778004 } ] }, "behavior": { "score": 5, "ruleId": "B_CLEAN_HIGH_INTERVENTION", "summary": "No red flags or missing roles were observed, but the team needed at least three interventions.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 5 } ] } }, "composite": 1.2, "band": "framework-gap" } }, { "suiteId": "productivity", "scenarioId": "craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7", "runDir": "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7", "finishedAt": "2026-10-28T11:13:39.977Z", "rawSuccess": true, "rawReason": "craftbook-ab-test-readout passed 19 deterministic craftbook checks", "durationMs": 287936, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 0.25, "judgeArtifactCandidates": [ "workspace/a-b-test-readout-eval/tasks/eval/readout.md", "artifacts/a-b-test-readout-eval/tasks/eval/readout.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 124.5, "nativePrefillTokensPerSec": 1859.14, "inputTokens": 105901, "outputTokens": 16824, "peakRssMb": 21095, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 287936, "derived": { "durationMs": 279476, "meanTokensPerSec": 124.5, "genTokensPerSec": 124.5, "genTokens": 16824, "promptTokensPerSec": 1859.14, "peakRssMb": 21095, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 101103, "output": 3243 } } }, "usage": { "available": true, "totalInputTokens": 105901, "totalOutputTokens": 16824, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 101103, "todayTokensCached": 98156, "todayTokensOut": 3243, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 101103, "totalTokensCached": 98156, "totalTokensOut": 3243, "totalCost": 0, "medianOutputTokensPerSec": 125.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 125.1, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10844, "cachedInputTokens": 10753, "outputTokens": 38, "cost": 0, "durationMs": 14469, "outputTokensPerSec": 86.78438068789522, "at": "2026-10-28T11:13:33.613Z" }, "lastUpdated": "2026-10-28T11:13:33.613Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:13:35.208Z" } }, "source": "engine-log" }, "process": { "sampleCount": 51, "peakRssMb": 21095, "peakCpuPercent": 264.7, "firstAtMs": 0, "lastAtMs": 278355 }, "gpu": { "available": true, "sampleCount": 51, "peakUtilPercent": 100, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 51, "peakUsedMb": 31445, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "judgmentAggregateEligibility": "excluded-rubric-mismatch", "judgmentRubricLimitation": "The frozen safety/mapping/signals/containment descriptions concern dry-run credentials, fake endpoints, retry/abort and mock-service containment. The requested A/B analysis contains no simulator plan. Low literal scores do not measure experiment-analysis quality or prove unsafe behavior.", "startedAt": "2026-10-28T11:08:52.041Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T11:20:58.637Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete required workspace readout, source brief, CSV and final stats.json read under the combined 20 KiB cap. Exact frozen simulator axes applied literally, with their scores excluded from qualitative aggregates because they do not measure A/B analysis. Deterministic gates and performance remain eligible.", "suiteId": "productivity", "scenarioId": "craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7", "artifactPath": "workspace/a-b-test-readout-eval/tasks/eval/readout.md", "contextArtifactPaths": [ "workspace/a-b-test-readout-eval/source/experiment-brief.md", "workspace/a-b-test-readout-eval/data/checkout-test.csv", "workspace/a-b-test-readout-eval/data/stats.json" ], "artifactKind": "markdown", "aggregateEligibility": "excluded-rubric-mismatch", "rubricLimitation": "The frozen safety/mapping/signals/containment descriptions concern dry-run credentials, fake endpoints, retry/abort and mock-service containment. The requested A/B analysis contains no simulator plan. Low literal scores do not measure experiment-analysis quality or prove unsafe behavior.", "taskArtifactReview": "Correctly leads with Do not ship after a copied tool-evidence preamble, retains all seeded counts/rates, +1.25 pp conversion lift, +10.42% relative lift, p=0.0078 and the +0.23 pp guardrail breach. It marks the absent confidence interval unverified instead of inventing one. The raw read_artifact instruction at the top and competing meanings of citation [1] reduce clarity; 'confirming' the badge drives conversions is too categorical. No replacement numeric task-quality rubric was invented.", "scoreAxes": { "safety": 0, "mapping": 0, "signals": 1, "containment": 0 }, "justification": "Rubric mismatch: the readout defines ship/no-ship conditions but no simulator dry-run boundary, endpoint mapping, retry/abort policy or containment plan. Exclude these literal scores from qualitative aggregates. Its actual A/B figures and guardrail decision are correct; copied tool-evidence prose and conflicting citation labels weaken presentation.", "artifactSha256": "f38559d0036642f1c703e92ec74499731df8467a47bf89c722565f2d2b4d7d7b", "artifactBytes": 2935, "contextArtifacts": [ { "path": "workspace/a-b-test-readout-eval/source/experiment-brief.md", "bytes": 504, "sha256": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed" }, { "path": "workspace/a-b-test-readout-eval/data/checkout-test.csv", "bytes": 95, "sha256": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35" }, { "path": "workspace/a-b-test-readout-eval/data/stats.json", "bytes": 438, "sha256": "f2d50a862f1ff5922b62b62ac4883aa4e19ac25000d3df6263c5597a84de8113" } ], "inputBytes": 3972, "inputLimitBytes": 20480, "truncated": false, "meanScore": 0.25 }, "systemMemory": { "sampleCount": 51, "peakUsedMb": 31445, "totalMb": 31865 }, "evidenceHashes": { "result.json": "d73c8de62fa898163ea554f524906bcc7429a96bb0add79a77dfd4085ac5841f", "score.json": "eba918496b1a4f4e175827af5cfcff260d60e0d4646b2409d9717535b7f2f7a1", "metrics.json": "f3136b1d5e0c7106edeae25e161c40fecf3080711f7307ac2b7acdb66b5c0851", "codex-inline-judge.json": "45bc28a2d709aee923914e179ca99f3d0cd587c61381e9be667d29dc6b5de25c" }, "rubricId": "craftbook-ab-test-readout", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7", "scenarioId": "craftbook-ab-test-readout", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33070 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-ab-test-readout", "bytes": 438, "score": 19, "scoreMax": 19, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.10664296296296297 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 35 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33070 }, { "source": "facts", "path": "outcome.durationMs", "value": 287936 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn", "runDir": "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn", "finishedAt": "2026-10-28T11:16:31.978Z", "rawSuccess": true, "rawReason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "durationMs": 167769, "rawFixedComposite": 9, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.5, "judgeArtifactCandidates": [ "workspace/annotated-bibliography-eval/bibliography.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 122.2, "nativePrefillTokensPerSec": 1857.89, "inputTokens": 66240, "outputTokens": 7645, "peakRssMb": 20655, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 167769, "derived": { "durationMs": 158087, "meanTokensPerSec": 122.2, "genTokensPerSec": 122.2, "genTokens": 7645, "promptTokensPerSec": 1857.89, "peakRssMb": 20655, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 66182, "output": 1690 } } }, "usage": { "available": true, "totalInputTokens": 66240, "totalOutputTokens": 7645, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 66182, "todayTokensCached": 64555, "todayTokensOut": 1690, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 66182, "totalTokensCached": 64555, "totalTokensOut": 1690, "totalCost": 0, "medianOutputTokensPerSec": 122.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 122.2, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11270, "cachedInputTokens": 10988, "outputTokens": 122, "cost": 0, "durationMs": 18399, "outputTokensPerSec": 120.45290291496025, "at": "2026-10-28T11:16:22.098Z" }, "lastUpdated": "2026-10-28T11:16:22.098Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:16:25.685Z" } }, "source": "engine-log" }, "process": { "sampleCount": 29, "peakRssMb": 20655, "peakCpuPercent": 260.7, "firstAtMs": 0, "lastAtMs": 157037 }, "gpu": { "available": true, "sampleCount": 29, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 30911, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:13:44.209Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T11:20:58.649Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 4336-byte bibliography and all six source cards read against the original prompt and exact accuracy/grounding/structure/actionability descriptions under the combined 20 KiB cap. Numeric accuracy assesses the supplied quantities and years; no new computation was required or claimed. This is a review of a synthetic eval fixture, with no external source verification or medical advice.", "suiteId": "productivity", "scenarioId": "craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn", "artifactPath": "workspace/annotated-bibliography-eval/bibliography.md", "contextArtifactPaths": [ "workspace/annotated-bibliography-eval/source/source-cards.md" ], "artifactKind": "markdown", "scoreAxes": { "accuracy": 8, "grounding": 5, "structure": 7, "actionability": 6 }, "justification": "All six APA-style entries preserve the supplied dates and figures and consistently separate summary, evaluation and relevance. However, 'primary barrier', 'high internal validity', and claims of necessity/effectiveness exceed the cards. Relevance is clear, but practical program choices remain general and several evaluations add unsupported confidence.", "artifactSha256": "e33d53b9808dea6017783a88fa772fcb3a75064bcb9df51ce42be2e3220077a8", "artifactBytes": 4336, "contextArtifacts": [ { "path": "workspace/annotated-bibliography-eval/source/source-cards.md", "bytes": 1294, "sha256": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7" } ], "inputBytes": 5630, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.5 }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 30911, "totalMb": 31865 }, "evidenceHashes": { "result.json": "c3d638888e0e1aa345f8e70beca18a2cf7841cc0072658d6a5e4802752ed8528", "score.json": "4c713dc428dc66d3b70c3fd5472516464c9762ed0e8470f21d9253c56d7299a0", "metrics.json": "5ab2d482ebfbf1a0a6c366e2406191b915e64658f7101b7b34faf7928a8b66c8", "codex-inline-judge.json": "7b743ca621c8d3073628d4d94fbea3dbc236cd57c06370c17f47d01afb5a24c6" }, "rubricId": "craftbook-annotated-bibliography", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn", "scenarioId": "craftbook-annotated-bibliography", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26821 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-annotated-bibliography", "bytes": 4336, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.062136666666666666 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 26 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26821 }, { "source": "facts", "path": "outcome.durationMs", "value": 167769 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts", "runDir": "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts", "finishedAt": "2026-10-28T11:19:33.209Z", "rawSuccess": true, "rawReason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "durationMs": 180515, "rawFixedComposite": 9, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.75, "judgeArtifactCandidates": [ "workspace/annotated-bibliography-eval/bibliography.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 120.44, "nativePrefillTokensPerSec": 1828.18, "inputTokens": 65338, "outputTokens": 8016, "peakRssMb": 21228, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 180515, "derived": { "durationMs": 174106, "meanTokensPerSec": 120.44, "genTokensPerSec": 120.44, "genTokens": 8016, "promptTokensPerSec": 1828.18, "peakRssMb": 21228, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 65287, "output": 1582 } } }, "usage": { "available": true, "totalInputTokens": 65338, "totalOutputTokens": 8016, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 65287, "todayTokensCached": 63053, "todayTokensOut": 1582, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 65287, "totalTokensCached": 63053, "totalTokensOut": 1582, "totalCost": 0, "medianOutputTokensPerSec": 110.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 110.9, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11196, "cachedInputTokens": 10909, "outputTokens": 122, "cost": 0, "durationMs": 17941, "outputTokensPerSec": 108.89109270861644, "at": "2026-10-28T11:19:25.171Z" }, "lastUpdated": "2026-10-28T11:19:25.171Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:19:30.073Z" } }, "source": "engine-log" }, "process": { "sampleCount": 32, "peakRssMb": 21228, "peakCpuPercent": 262.9, "firstAtMs": 0, "lastAtMs": 173033 }, "gpu": { "available": true, "sampleCount": 32, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 31314, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:16:32.694Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T11:22:03.044Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 3674-byte bibliography and all six seeded cards read against the original prompt and exact accuracy/grounding/structure/actionability descriptions under the combined 20 KiB cap. Numeric accuracy assesses the supplied quantities and dates; no new calculation was required or claimed. Synthetic fixture reviewed without external source verification.", "suiteId": "productivity", "scenarioId": "craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts", "artifactPath": "workspace/annotated-bibliography-eval/bibliography.md", "contextArtifactPaths": [ "workspace/annotated-bibliography-eval/source/source-cards.md" ], "artifactKind": "markdown", "scoreAxes": { "accuracy": 8, "grounding": 5, "structure": 8, "actionability": 6 }, "justification": "Preserves all six sources and supplied figures, with useful thematic groups and uniform annotations. Calling connectivity a primary reason, the pilot highly credible, and the privacy review foundational adds strength absent from the cards. Relevance is easy to follow, but recommendations stay broad and citation titles use inconsistent APA capitalization.", "artifactSha256": "b033db0715fc2fc3bc0417e88fa0fa1f7978dd068efe6c7479e84e6092e5ebca", "artifactBytes": 3674, "contextArtifacts": [ { "path": "workspace/annotated-bibliography-eval/source/source-cards.md", "bytes": 1294, "sha256": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7" } ], "inputBytes": 4968, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.75 }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 31314, "totalMb": 31865 }, "evidenceHashes": { "result.json": "563fd93262ff94371e73899a763da987c1e6390f51ae7b7b577138d0e713fa52", "score.json": "5215b66b227a2acd6aae9d639fa6374a832bfe47932c9bb0505624cf5ffd9f85", "metrics.json": "2e703efa1bd2db7c9dc4ef15a03782b2ee16d0b727cb7f3eabdac43fcfa8f2f6", "codex-inline-judge.json": "5bfb2cfbff1e8801575617643180d2b2a0fde8965b7d6aa951c329f8192bfc8e" }, "rubricId": "craftbook-annotated-bibliography", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts", "scenarioId": "craftbook-annotated-bibliography", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 28240 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-annotated-bibliography", "bytes": 3674, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06685740740740741 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 26 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 28240 }, { "source": "facts", "path": "outcome.durationMs", "value": 180515 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq", "runDir": "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq", "finishedAt": "2026-10-28T11:21:20.301Z", "rawSuccess": true, "rawReason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "durationMs": 106513, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6, "judgeArtifactCandidates": [ "workspace/annotated-bibliography-eval/bibliography.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 122.68, "nativePrefillTokensPerSec": 1794.85, "inputTokens": 41393, "outputTokens": 3914, "peakRssMb": 21058, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 106513, "derived": { "durationMs": 95548, "meanTokensPerSec": 122.68, "genTokensPerSec": 122.68, "genTokens": 3914, "promptTokensPerSec": 1794.85, "peakRssMb": 21058, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 41373, "output": 1236 } } }, "usage": { "available": true, "totalInputTokens": 41393, "totalOutputTokens": 3914, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 41373, "todayTokensCached": 40195, "todayTokensOut": 1236, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 41373, "totalTokensCached": 40195, "totalTokensOut": 1236, "totalCost": 0, "medianOutputTokensPerSec": 118.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 118.5, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10780, "cachedInputTokens": 10525, "outputTokens": 126, "cost": 0, "durationMs": 18362, "outputTokensPerSec": 117.2019671178161, "at": "2026-10-28T11:21:11.651Z" }, "lastUpdated": "2026-10-28T11:21:11.651Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:21:12.743Z" } }, "source": "engine-log" }, "process": { "sampleCount": 18, "peakRssMb": 21058, "peakCpuPercent": 263.2, "firstAtMs": 1, "lastAtMs": 94504 }, "gpu": { "available": true, "sampleCount": 18, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 30851, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:19:33.788Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T11:22:55.575Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 3433-byte bibliography and its six seeded source cards read against the original prompt and exact accuracy/grounding/structure/actionability descriptions under the combined 20 KiB cap. Numeric accuracy assesses supplied quantities and dates; no new calculation was requested or claimed. Synthetic fixture reviewed without external source verification.", "suiteId": "productivity", "scenarioId": "craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq", "artifactPath": "workspace/annotated-bibliography-eval/bibliography.md", "contextArtifactPaths": [ "workspace/annotated-bibliography-eval/source/source-cards.md" ], "artifactKind": "markdown", "scoreAxes": { "accuracy": 8, "grounding": 4, "structure": 7, "actionability": 5 }, "justification": "All six sources, dates and figures survive in consistent annotations. But it adds statistical significance, legal requirements and a claim that no follow-up program works without reliable internet, despite the card discussing phone/in-person options. APA title capitalization is inconsistent; relevance is clear but practical guidance remains generic.", "artifactSha256": "5330becfbcae1b2a497ee8fbef127416db53aa7fa5a92e09badb15e1f757d645", "artifactBytes": 3433, "contextArtifacts": [ { "path": "workspace/annotated-bibliography-eval/source/source-cards.md", "bytes": 1294, "sha256": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7" } ], "inputBytes": 4727, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6 }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 30851, "totalMb": 31865 }, "evidenceHashes": { "result.json": "6817b2d518aebf9672e759df9a57fd17ab8379fb0aa00c116d1b77d3c0232751", "score.json": "ba541f03375e0c28bfa729d5025980521d1a2dccf0151a40e87b4044d488e825", "metrics.json": "d73f9e269cf866fe78f1189e09aa77c0a715d7bc8e54a747cd7a0eed44db46f1", "codex-inline-judge.json": "152b90bc05d87b22d3d8be99460748a1714fe302bf84a31a2d2c6519bc013bd8" }, "rubricId": "craftbook-annotated-bibliography", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq", "scenarioId": "craftbook-annotated-bibliography", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 28078 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-annotated-bibliography", "bytes": 3433, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.03944925925925926 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 14 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 28078 }, { "source": "facts", "path": "outcome.durationMs", "value": 106513 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-research-to-document", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0", "runDir": "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0", "finishedAt": "2026-10-28T11:24:17.841Z", "rawSuccess": true, "rawReason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "durationMs": 173220, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6, "judgeArtifactCandidates": [ "workspace/word-document-from-content-eval/documents/eval/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 122.51, "nativePrefillTokensPerSec": 1809.96, "inputTokens": 77247, "outputTokens": 4318, "peakRssMb": 21488, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 173220, "derived": { "durationMs": 158787, "meanTokensPerSec": 122.51, "genTokensPerSec": 122.51, "genTokens": 4318, "promptTokensPerSec": 1809.96, "peakRssMb": 21488, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 68302, "output": 674 } } }, "usage": { "available": true, "totalInputTokens": 77247, "totalOutputTokens": 4318, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 68302, "todayTokensCached": 67318, "todayTokensOut": 674, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 68302, "totalTokensCached": 67318, "totalTokensOut": 674, "totalCost": 0, "medianOutputTokensPerSec": 120, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 120, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18521, "cachedInputTokens": 18255, "outputTokens": 35, "cost": 0, "durationMs": 30804, "outputTokensPerSec": 120.03403317881894, "at": "2026-10-28T11:24:01.537Z" }, "lastUpdated": "2026-10-28T11:24:01.537Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:24:06.922Z" } }, "source": "engine-log" }, "process": { "sampleCount": 29, "peakRssMb": 21488, "peakCpuPercent": 258.2, "firstAtMs": 0, "lastAtMs": 157694 }, "gpu": { "available": true, "sampleCount": 29, "peakUtilPercent": 99, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 31375, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:21:24.621Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T11:31:51.403Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 1361-byte Markdown report and seeded 405-byte brief read against the original prompt and exact grounding/structure/convertibility axes under the combined 20 KiB cap. Recorded project history confirms successful conversion, preview and final save; destination-schema errors were repaired. No separate visual DOCX review or fresh rendering was performed.", "suiteId": "productivity", "scenarioId": "craftbook-research-to-document", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0", "artifactPath": "workspace/word-document-from-content-eval/documents/eval/report.md", "contextArtifactPaths": [ "workspace/word-document-from-content-eval/source/brief.md" ], "artifactKind": "markdown", "scoreAxes": { "grounding": 4, "structure": 7, "convertibility": 7 }, "justification": "Includes every supplied metric/action in a clear title, summary, table and findings. But it adds an imprecise 66% reduction despite the no-other-metrics instruction, labels the 5.3-point leakage change as percent, and gives [2] conflicting source mappings. Complete plain Markdown is suitable for conversion; the recorded publish flow succeeds.", "artifactSha256": "ba577678cc81326d6e368821211e7140a3e13c6d82847dc3adf885e84e764c65", "artifactBytes": 1361, "contextArtifacts": [ { "path": "workspace/word-document-from-content-eval/source/brief.md", "bytes": 405, "sha256": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9" } ], "inputBytes": 1766, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6 }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 31375, "totalMb": 31865 }, "evidenceHashes": { "result.json": "0ef1023c69ba9fa620bda559b84ed6ad93d14a0961f4e66da23f2fdef466015d", "score.json": "d131c2d536dd3f21e5f7aef68fe6a58324716918193ea2719ae27fdd77b0e2d1", "metrics.json": "821ab9217e52b433aef7200618b3f52770c8235b7aa63857387ae682b3f11b16", "codex-inline-judge.json": "6f66357668b8f3d98ec39124b4b32acb578591480a717a8169bcf060bf45cdc9" }, "rubricId": "craftbook-research-to-document", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0", "scenarioId": "craftbook-research-to-document", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26634 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-research-to-document", "bytes": 1361, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06415555555555555 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 30 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26634 }, { "source": "facts", "path": "outcome.durationMs", "value": 173220 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-research-to-document", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8", "runDir": "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8", "finishedAt": "2026-10-28T11:28:51.162Z", "rawSuccess": true, "rawReason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "durationMs": 272264, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6.333333333333333, "judgeArtifactCandidates": [ "workspace/word-document-from-content-eval/documents/eval/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 119.82, "nativePrefillTokensPerSec": 1747.96, "inputTokens": 146417, "outputTokens": 9899, "peakRssMb": 21401, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 272264, "derived": { "durationMs": 262899, "meanTokensPerSec": 119.82, "genTokensPerSec": 119.82, "genTokens": 9899, "promptTokensPerSec": 1747.96, "peakRssMb": 21401, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 109789, "output": 1312 } } }, "usage": { "available": true, "totalInputTokens": 146417, "totalOutputTokens": 9899, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 109789, "todayTokensCached": 107541, "todayTokensOut": 1312, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 109789, "totalTokensCached": 107541, "totalTokensOut": 1312, "totalCost": 0, "medianOutputTokensPerSec": 119.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 119.6, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19513, "cachedInputTokens": 19166, "outputTokens": 35, "cost": 0, "durationMs": 29150, "outputTokensPerSec": 125.82666266981973, "at": "2026-10-28T11:28:44.227Z" }, "lastUpdated": "2026-10-28T11:28:44.227Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:28:45.551Z" } }, "source": "engine-log" }, "process": { "sampleCount": 48, "peakRssMb": 21401, "peakCpuPercent": 257.5, "firstAtMs": 1, "lastAtMs": 261877 }, "gpu": { "available": true, "sampleCount": 48, "peakUtilPercent": 100, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 48, "peakUsedMb": 31221, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:24:18.898Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T11:31:51.411Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 1292-byte Markdown report and seeded 405-byte brief read against the original prompt and exact grounding/structure/convertibility axes under the combined 20 KiB cap. Project history records initial unreadable-DOCX preview, then successful conversion, replacement save at 11:28:07 and preview at 11:28:38. No fresh renderer or visual DOCX judgment was used.", "suiteId": "productivity", "scenarioId": "craftbook-research-to-document", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8", "artifactPath": "workspace/word-document-from-content-eval/documents/eval/report.md", "contextArtifactPaths": [ "workspace/word-document-from-content-eval/source/brief.md" ], "artifactKind": "markdown", "scoreAxes": { "grounding": 5, "structure": 7, "convertibility": 7 }, "justification": "Preserves the brief's figures, unresolved complaint and three actions in a usable compact report. Adds derived improvement metrics despite the instruction, labels a 5.3-percentage-point change as 5.3%, and introduces scaling/significance claims. Markdown is complete and conversion-ready; recorded repairs produce a successfully previewed DOCX.", "artifactSha256": "5e6c0262e9c07beb3fd102cdbab68c6ce55d96d4d450654fe6fa8974ee8910d8", "artifactBytes": 1292, "contextArtifacts": [ { "path": "workspace/word-document-from-content-eval/source/brief.md", "bytes": 405, "sha256": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9" } ], "inputBytes": 1697, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6.333333333333333 }, "systemMemory": { "sampleCount": 48, "peakUsedMb": 31221, "totalMb": 31865 }, "evidenceHashes": { "result.json": "46708580c247e5af5289d9c3974429451832be2f0c35d761422d4367376d5a06", "score.json": "d13e3e8a471fcb4ce4305ffb31ee60015986a9e2d5352f4bcac41486dec47058", "metrics.json": "7a979221622012bf9b02d7364e9b4e46eebb43a804f379f565b9d3c09c28e71f", "codex-inline-judge.json": "8b32f248d4ab00f3aa280b2071143e7dc1d9a6c4aceead4d0a9ba599d01d30f1" }, "rubricId": "craftbook-research-to-document", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8", "scenarioId": "craftbook-research-to-document", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26737 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-research-to-document", "bytes": 1292, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.10083851851851852 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 66 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26737 }, { "source": "facts", "path": "outcome.durationMs", "value": 272264 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "productivity", "scenarioId": "craftbook-research-to-document", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn", "runDir": "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn", "finishedAt": "2026-10-28T11:31:26.038Z", "rawSuccess": true, "rawReason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "durationMs": 153504, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 6, "judgeArtifactCandidates": [ "workspace/word-document-from-content-eval/documents/eval/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 125.4, "nativePrefillTokensPerSec": 1826.48, "inputTokens": 92507, "outputTokens": 4397, "peakRssMb": 21566, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 153504, "derived": { "durationMs": 140368, "meanTokensPerSec": 125.4, "genTokensPerSec": 125.4, "genTokens": 4397, "promptTokensPerSec": 1826.48, "peakRssMb": 21566, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 69466, "output": 707 } } }, "usage": { "available": true, "totalInputTokens": 92507, "totalOutputTokens": 4397, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 69466, "todayTokensCached": 68412, "todayTokensOut": 707, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 69466, "totalTokensCached": 68412, "totalTokensOut": 707, "totalCost": 0, "medianOutputTokensPerSec": 126.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 126.6, "turns": 5 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18806, "cachedInputTokens": 18451, "outputTokens": 41, "cost": 0, "durationMs": 27438, "outputTokensPerSec": 126.59509823779622, "at": "2026-10-28T11:31:15.635Z" }, "lastUpdated": "2026-10-28T11:31:15.635Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:31:16.136Z" } }, "source": "engine-log" }, "process": { "sampleCount": 26, "peakRssMb": 21566, "peakCpuPercent": 241.2, "firstAtMs": 0, "lastAtMs": 139313 }, "gpu": { "available": true, "sampleCount": 26, "peakUtilPercent": 100, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 31182, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:28:52.534Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T11:32:33.154Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete 1299-byte Markdown report and seeded 405-byte brief read against the original prompt and exact grounding/structure/convertibility axes under the combined 20 KiB cap. Captured project history confirms conversion, preview, repaired destination arguments and final save at 11:30:41, followed by successful preview at 11:31:10. No separate visual DOCX review or new renderer was used.", "suiteId": "productivity", "scenarioId": "craftbook-research-to-document", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn", "artifactPath": "workspace/word-document-from-content-eval/documents/eval/report.md", "contextArtifactPaths": [ "workspace/word-document-from-content-eval/source/brief.md" ], "artifactKind": "markdown", "scoreAxes": { "grounding": 4, "structure": 7, "convertibility": 7 }, "justification": "Preserves all supplied facts and actions with clear sections, table and source mappings. However, the added 66% reduction is imprecise, the 5.3-point change is mislabeled percent, and achieved performance goals are not in the brief. Complete plain Markdown converts successfully through the recorded publishing workflow.", "artifactSha256": "5fe096339142c5c5dbc8d42932e6dbfe125039a2542de2673b4a6dfeff668630", "artifactBytes": 1299, "contextArtifacts": [ { "path": "workspace/word-document-from-content-eval/source/brief.md", "bytes": 405, "sha256": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9" } ], "inputBytes": 1704, "inputLimitBytes": 20480, "truncated": false, "meanScore": 6 }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 31182, "totalMb": 31865 }, "evidenceHashes": { "result.json": "acd30474bb713e1a869219a25b39ee8869dbe4ae209620ff36f3bdd3605c084c", "score.json": "e961a7060f3f74c9f860318e3c5e25d9091e391123e44b9f9bbf9700ae4b3a7d", "metrics.json": "06f2fbe6d9d7e7061fe5dcaa8889e67a28b99b7a8bc536e7e8bbf449d1c15d78", "codex-inline-judge.json": "53fe99cff4f3a81c0990593ce8115b2e575e78c848b355ce7c9ded6c37304def" }, "rubricId": "craftbook-research-to-document", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn", "scenarioId": "craftbook-research-to-document", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 27807 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-research-to-document", "bytes": 1299, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.05685333333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 31 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 27807 }, { "source": "facts", "path": "outcome.durationMs", "value": 153504 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "fix-squisq-bugs", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx", "runDir": "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx", "finishedAt": "2026-10-28T11:34:12.102Z", "rawSuccess": true, "rawReason": "all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)", "durationMs": 159230, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 137.13, "nativePrefillTokensPerSec": 1886.59, "inputTokens": 86640, "outputTokens": 11714, "peakRssMb": 16159, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 159230, "derived": { "durationMs": 151984, "meanTokensPerSec": 137.13, "genTokensPerSec": 137.13, "genTokens": 11714, "promptTokensPerSec": 1886.59, "peakRssMb": 16159, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 53835, "output": 1767 } } }, "usage": { "available": true, "totalInputTokens": 86640, "totalOutputTokens": 11714, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 53835, "todayTokensCached": 33790, "todayTokensOut": 1767, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 53835, "totalTokensCached": 33790, "totalTokensOut": 1767, "totalCost": 0, "medianOutputTokensPerSec": 108.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 108.5, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 21979, "cachedInputTokens": 20400, "outputTokens": 59, "cost": 0, "durationMs": 41067, "outputTokensPerSec": 108.46859179715625, "at": "2026-10-28T11:34:03.374Z" }, "lastUpdated": "2026-10-28T11:34:03.374Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:34:07.853Z" } }, "source": "engine-log" }, "process": { "sampleCount": 28, "peakRssMb": 16159, "peakCpuPercent": 268.1, "firstAtMs": 3, "lastAtMs": 150450 }, "gpu": { "available": true, "sampleCount": 28, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 26375, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:31:32.872Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:41:03.342Z", "suiteId": "developer", "scenarioId": "fix-squisq-bugs", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 26375, "totalMb": 31865 }, "evidenceHashes": { "result.json": "3afa2fc9025d9451cbfd21f622c7fdb66f0cab420d24b498627fb29a0302b66f", "score.json": "60567f955d1a3b3f26b2962931c239c0a3f46501a73ef57b1cf23ae631ea95c3", "metrics.json": "c36f488c4817d6ec9a45eb71921f5a0e92d01d4ff9c3dc2db9768353328c9106", "codex-inline-judge.json": "dcdd52999f3f7bb04a5c3c83713df25bebfe30d3d4dc2f5b966b51c779634076" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx", "scenarioId": "fix-squisq-bugs", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23258 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "fix-squisq-bugs", "bytes": 6188, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.10615333333333334 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 14 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23258 }, { "source": "facts", "path": "outcome.durationMs", "value": 159230 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "fix-squisq-bugs", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht", "runDir": "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht", "finishedAt": "2026-10-28T11:35:21.709Z", "rawSuccess": true, "rawReason": "all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)", "durationMs": 68553, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 139.43, "nativePrefillTokensPerSec": 1649.04, "inputTokens": 15698, "outputTokens": 5451, "peakRssMb": 15897, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 68553, "derived": { "durationMs": 62069, "meanTokensPerSec": 139.43, "genTokensPerSec": 139.43, "genTokens": 5451, "promptTokensPerSec": 1649.04, "peakRssMb": 15897, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15653, "output": 214 } } }, "usage": { "available": true, "totalInputTokens": 15698, "totalOutputTokens": 5451, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 15653, "todayTokensCached": 14955, "todayTokensOut": 214, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 15653, "totalTokensCached": 14955, "totalTokensOut": 214, "totalCost": 0, "medianOutputTokensPerSec": 124.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 124.5, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15653, "cachedInputTokens": 14955, "outputTokens": 214, "cost": 0, "durationMs": 57832, "outputTokensPerSec": 124.52768636328292, "at": "2026-10-28T11:35:16.531Z" }, "lastUpdated": "2026-10-28T11:35:16.531Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:35:18.469Z" } }, "source": "engine-log" }, "process": { "sampleCount": 12, "peakRssMb": 15897, "peakCpuPercent": 261.9, "firstAtMs": 1, "lastAtMs": 60757 }, "gpu": { "available": true, "sampleCount": 12, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 12, "peakUsedMb": 26037, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:34:13.156Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:41:03.355Z", "suiteId": "developer", "scenarioId": "fix-squisq-bugs", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 12, "peakUsedMb": 26037, "totalMb": 31865 }, "evidenceHashes": { "result.json": "c23a56ca8f207cb759f97ef6abcdc7ec850df1fe9b1555036e19c50f6749b148", "score.json": "9b593a7328bf57fb46e88a810fa0fefd3f50f300630065127bd83447c5640d9d", "metrics.json": "67e620cd1fb1494411533afc8cd56765e5c8c2a5f358037c2db0259533dd39fa", "codex-inline-judge.json": "6d84c74b3c6d4892932c6da51da57bea36e27922ef4ccf94a3c9c0390c734995" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht", "scenarioId": "fix-squisq-bugs", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25512 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "fix-squisq-bugs", "bytes": 6610, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.045702 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 11 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 25512 }, { "source": "facts", "path": "outcome.durationMs", "value": 68553 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "fix-squisq-bugs", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy", "runDir": "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy", "finishedAt": "2026-10-28T11:42:58.310Z", "rawSuccess": true, "rawReason": "all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)", "durationMs": 455499, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 132.02, "nativePrefillTokensPerSec": 1776.75, "inputTokens": 387996, "outputTokens": 26318, "peakRssMb": 17199, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 455499, "derived": { "durationMs": 445332, "meanTokensPerSec": 132.02, "genTokensPerSec": 132.02, "genTokens": 26318, "promptTokensPerSec": 1776.75, "peakRssMb": 17199, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 194457, "output": 9623 } } }, "usage": { "available": true, "totalInputTokens": 387996, "totalOutputTokens": 26318, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 194457, "todayTokensCached": 30398, "todayTokensOut": 9623, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 194457, "totalTokensCached": 30398, "totalTokensOut": 9623, "totalCost": 0, "medianOutputTokensPerSec": 143.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 143.5, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 41347, "outputTokens": 726, "cost": 0, "durationMs": 31514, "outputTokensPerSec": 113.23745895142112, "at": "2026-10-28T11:42:48.202Z" }, "lastUpdated": "2026-10-28T11:42:48.202Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:42:51.062Z" } }, "source": "engine-log" }, "process": { "sampleCount": 81, "peakRssMb": 17199, "peakCpuPercent": 272, "firstAtMs": 0, "lastAtMs": 444049 }, "gpu": { "available": true, "sampleCount": 81, "peakUtilPercent": 100, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 81, "peakUsedMb": 28420, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:35:22.811Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:48:01.912Z", "suiteId": "developer", "scenarioId": "fix-squisq-bugs", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 81, "peakUsedMb": 28420, "totalMb": 31865 }, "evidenceHashes": { "result.json": "3277026b786fb9f0c748012849528ca5c5f64686059fc6c52d7624dcaee71194", "score.json": "f2b92cedd390f3ad17f2c0b228c9d55f7aba9018ffc8425b7c3ff2ed48fe96be", "metrics.json": "e2ed8467538fec2cc1f83263326d931c70f1a0ca5c552fda87beca37b3fc99d2", "codex-inline-judge.json": "66bf2d35ed95fccd3cd27a0162e18dc9f9a8e1f6bdb3e3b4257341d0ef9c12bf" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy", "scenarioId": "fix-squisq-bugs", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26384 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "fix-squisq-bugs", "bytes": 6638, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.303666 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 30 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26384 }, { "source": "facts", "path": "outcome.durationMs", "value": 455499 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "dev-craftbook-routing", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9", "runDir": "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9", "finishedAt": "2026-10-28T11:47:37.097Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook selection (engineering); missing signals stayed unchanged: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it, a new craftbook was authored (regression-test-and-fix) — the library already covers this; use the existing recipe instead. Last failure: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it", "durationMs": 274213, "rawFixedComposite": 2, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 111.46, "nativePrefillTokensPerSec": 1855.5, "inputTokens": 153632, "outputTokens": 13179, "peakRssMb": 19012, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 274213, "derived": { "durationMs": 266903, "meanTokensPerSec": 111.46, "genTokensPerSec": 111.46, "genTokens": 13179, "promptTokensPerSec": 1855.5, "peakRssMb": 19012, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 173149, "output": 4157 } } }, "usage": { "available": true, "totalInputTokens": 153632, "totalOutputTokens": 13179, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 173149, "todayTokensCached": 122596, "todayTokensOut": 4157, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 173149, "totalTokensCached": 122596, "totalTokensOut": 4157, "totalCost": 0, "medianOutputTokensPerSec": 103, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 103, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17168, "outputTokens": 135, "cost": 0, "durationMs": 24059, "outputTokensPerSec": 123.47465037784164, "at": "2026-10-28T11:47:29.458Z" }, "lastUpdated": "2026-10-28T11:47:29.458Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:47:33.473Z" } }, "source": "engine-log" }, "process": { "sampleCount": 49, "peakRssMb": 19012, "peakCpuPercent": 295.2, "firstAtMs": 0, "lastAtMs": 265880 }, "gpu": { "available": true, "sampleCount": 49, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 29652, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:43:02.884Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:48:01.891Z", "suiteId": "developer", "scenarioId": "dev-craftbook-routing", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 29652, "totalMb": 31865 }, "evidenceHashes": { "result.json": "99e7f27efc8479699afdd7cddc290e85a46a4060a354c376151c03c74a704d3a", "score.json": "a73caaaf840060e6f7453cda6bea7bdd607fd5434eee2e2b4b42da18296588b4", "metrics.json": "4f4af9bb89f83060acc6ab28ebd2a6fef7f130e70bd8e2694f961b76071fe8f1", "codex-inline-judge.json": "389f78c0dea8fed76ee2f9c3b2961771f0869a88a6dfaaeb3b395390c7ba28d5" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9", "scenarioId": "dev-craftbook-routing", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook selection (engineering); missing signals stayed unchanged: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it, a new craftbook was authored (regression-test-and-fix) — the library already covers this; use the existing recipe instead. Last failure: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 113390 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 0, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } } ] }, "quality": { "score": 0, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 0% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 113390 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 0, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.18280866666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 37 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 113390 }, { "source": "facts", "path": "outcome.durationMs", "value": 274213 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 2, "band": "framework-gap" } }, { "suiteId": "developer", "scenarioId": "dev-craftbook-routing", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n", "runDir": "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n", "finishedAt": "2026-10-28T11:53:02.634Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook selection (engineering); missing signals stayed unchanged: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it, a new craftbook was authored (regression-test-and-fix) — the library already covers this; use the existing recipe instead. Last failure: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it", "durationMs": 324398, "rawFixedComposite": 2, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 112, "nativePrefillTokensPerSec": 1775.15, "inputTokens": 210763, "outputTokens": 15813, "peakRssMb": 18916, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 324398, "derived": { "durationMs": 317444, "meanTokensPerSec": 112, "genTokensPerSec": 112, "genTokens": 15813, "promptTokensPerSec": 1775.15, "peakRssMb": 18916, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 243846, "output": 3643 } } }, "usage": { "available": true, "totalInputTokens": 210763, "totalOutputTokens": 15813, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 243846, "todayTokensCached": 131256, "todayTokensOut": 3643, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 243846, "totalTokensCached": 131256, "totalTokensOut": 3643, "totalCost": 0, "medianOutputTokensPerSec": 100, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 100, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 28812, "outputTokens": 1085, "cost": 0, "durationMs": 27675, "outputTokensPerSec": 95.5701052126351, "at": "2026-10-28T11:52:58.179Z" }, "lastUpdated": "2026-10-28T11:52:58.179Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:52:58.909Z" } }, "source": "engine-log" }, "process": { "sampleCount": 58, "peakRssMb": 18916, "peakCpuPercent": 287.5, "firstAtMs": 0, "lastAtMs": 316369 }, "gpu": { "available": true, "sampleCount": 58, "peakUtilPercent": 100, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 58, "peakUsedMb": 29525, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:47:38.236Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:59:49.390Z", "suiteId": "developer", "scenarioId": "dev-craftbook-routing", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 58, "peakUsedMb": 29525, "totalMb": 31865 }, "evidenceHashes": { "result.json": "eefad9d1b6d994eba09b54f1d7e4e07e9b6f96940009676b27402a08ea95b3ab", "score.json": "8b97ab835410b7a183f2ef9e299370b2906418f4c5f8c5a502822a6b45d8455d", "metrics.json": "08ab522e86b21901bb82a2b200d533b9237886ec8463d7546a2e87fce6d177b7", "codex-inline-judge.json": "acb04ac492da7407bc5ca3f10cabd614c30e41bec055507d3296ee563f47bbdf" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n", "scenarioId": "dev-craftbook-routing", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook selection (engineering); missing signals stayed unchanged: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it, a new craftbook was authored (regression-test-and-fix) — the library already covers this; use the existing recipe instead. Last failure: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 47036 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 0, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } } ] }, "quality": { "score": 0, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 0% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 47036 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 0, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.21626533333333334 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 52 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 47036 }, { "source": "facts", "path": "outcome.durationMs", "value": 324398 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 2, "band": "framework-gap" } }, { "suiteId": "developer", "scenarioId": "dev-craftbook-routing", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy", "runDir": "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy", "finishedAt": "2026-10-28T11:54:16.445Z", "rawSuccess": true, "rawReason": "task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "durationMs": 72717, "rawFixedComposite": 8.8, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 110.02, "nativePrefillTokensPerSec": 1635.34, "inputTokens": 12863, "outputTokens": 4012, "peakRssMb": 16668, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 72717, "derived": { "durationMs": 62151, "meanTokensPerSec": 110.02, "genTokensPerSec": 110.02, "genTokens": 4012, "promptTokensPerSec": 1635.34, "peakRssMb": 16668, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 24085, "output": 100 } } }, "usage": { "available": true, "totalInputTokens": 12863, "totalOutputTokens": 4012, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 24085, "todayTokensCached": 23581, "todayTokensOut": 100, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 24085, "totalTokensCached": 23581, "totalTokensOut": 100, "totalCost": 0, "medianOutputTokensPerSec": 90.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 90.6, "turns": 2 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12829, "cachedInputTokens": 12598, "outputTokens": 43, "cost": 0, "durationMs": 23919, "outputTokensPerSec": 86.07686253934943, "at": "2026-10-28T11:54:04.531Z" }, "lastUpdated": "2026-10-28T11:54:04.531Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:54:09.019Z" } }, "source": "engine-log" }, "process": { "sampleCount": 12, "peakRssMb": 16668, "peakCpuPercent": 287.1, "firstAtMs": 0, "lastAtMs": 60980 }, "gpu": { "available": true, "sampleCount": 12, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 12, "peakUsedMb": 27223, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:53:03.728Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:59:49.398Z", "suiteId": "developer", "scenarioId": "dev-craftbook-routing", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 12, "peakUsedMb": 27223, "totalMb": 31865 }, "evidenceHashes": { "result.json": "9568205d2554101ed213a2c89dbcffb5a295dff300b2b5b8da8cea38f3a22c79", "score.json": "e6b0a030914915955a2588d3ddda2ea3df3161328a0f9684902791ac5cc460c9", "metrics.json": "b85913545dc1580151e91f5c022a470d121d5896a1fe55c8927d2a4d3d3607ab", "codex-inline-judge.json": "1015614e18bd71cb65b40f5b007831504b13c0693497ee4c77254f48c3a7122b" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy", "scenarioId": "dev-craftbook-routing", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.048478 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 72717 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "interface-contract", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift", "runDir": "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift", "finishedAt": "2026-10-28T11:55:12.269Z", "rawSuccess": true, "rawReason": "tsc emit clean + node integration.mjs passed: 5 event(s) across 3 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "durationMs": 52574, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 124.85, "nativePrefillTokensPerSec": 1727.83, "inputTokens": 15557, "outputTokens": 1806, "peakRssMb": 16678, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 52574, "derived": { "durationMs": 40213, "meanTokensPerSec": 124.85, "genTokensPerSec": 124.85, "genTokens": 1806, "promptTokensPerSec": 1727.83, "peakRssMb": 16678, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7324, "output": 23 } } }, "usage": { "available": true, "totalInputTokens": 15557, "totalOutputTokens": 1806, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7324, "todayTokensCached": 7156, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7324, "totalTokensCached": 7156, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 81.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 81.2, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7324, "cachedInputTokens": 7156, "outputTokens": 23, "cost": 0, "durationMs": 16913, "outputTokensPerSec": 81.24826885790787, "at": "2026-10-28T11:54:42.457Z" }, "lastUpdated": "2026-10-28T11:54:42.457Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:55:03.393Z" } }, "source": "engine-log" }, "process": { "sampleCount": 8, "peakRssMb": 16678, "peakCpuPercent": 260, "firstAtMs": 1, "lastAtMs": 38921 }, "gpu": { "available": true, "sampleCount": 8, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 27328, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:54:19.695Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:59:49.437Z", "suiteId": "developer", "scenarioId": "interface-contract", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 27328, "totalMb": 31865 }, "evidenceHashes": { "result.json": "072fe180ecdcf9312b2e9eda497e86f6a567032d279a9cb6aa43e41c33988806", "score.json": "a46e4e961a2a32c2b31f276ff3d29d890f2c7aad08b64ba0443e75407434fc61", "metrics.json": "6c83d6bb7ab1a2690415e7cb1d0e02f1d20dff3494345f9da2bfdf8b8f0e5457", "codex-inline-judge.json": "c5730c2a244a0bf5f8e0db2e620275fc27f623c138f26119f957d8bf12b17249" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift", "scenarioId": "interface-contract", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 35014 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "interface-contract", "bytes": 1415, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.029207777777777777 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 35014 }, { "source": "facts", "path": "outcome.durationMs", "value": 52574 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "interface-contract", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw", "runDir": "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw", "finishedAt": "2026-10-28T11:56:02.907Z", "rawSuccess": true, "rawReason": "tsc emit clean + node integration.mjs passed: 5 event(s) across 3 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "durationMs": 49762, "rawFixedComposite": 10, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 115.11, "nativePrefillTokensPerSec": 1844.72, "inputTokens": 15007, "outputTokens": 1810, "peakRssMb": 14748, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 49762, "derived": { "durationMs": 40204, "meanTokensPerSec": 115.11, "genTokensPerSec": 115.11, "genTokens": 1810, "promptTokensPerSec": 1844.72, "peakRssMb": 14748, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7318, "output": 23 } } }, "usage": { "available": true, "totalInputTokens": 15007, "totalOutputTokens": 1810, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7318, "todayTokensCached": 7150, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7318, "totalTokensCached": 7150, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 86.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 86.1, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7318, "cachedInputTokens": 7150, "outputTokens": 23, "cost": 0, "durationMs": 21221, "outputTokensPerSec": 86.11174172740154, "at": "2026-10-28T11:55:39.972Z" }, "lastUpdated": "2026-10-28T11:55:39.972Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:55:56.624Z" } }, "source": "engine-log" }, "process": { "sampleCount": 8, "peakRssMb": 14748, "peakCpuPercent": 261.6, "firstAtMs": 1, "lastAtMs": 38845 }, "gpu": { "available": true, "sampleCount": 8, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 25375, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:55:13.145Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:59:49.445Z", "suiteId": "developer", "scenarioId": "interface-contract", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 25375, "totalMb": 31865 }, "evidenceHashes": { "result.json": "87c51ffd4c2287e05f0b597b04ad545d4fd56bd928e3d1a4bc111ce72cb11ace", "score.json": "22ae11a9081a948396aa11204187b94d425c3f75dde298af31bf9d3bab5b403c", "metrics.json": "b9270fcc3c602a39a0d86cc799ec46eddb6347e6c732af12fc0b2495eeb410ac", "codex-inline-judge.json": "c0eafe51ca91f094ff452497da43e4538217dbbf072736fbb94cff6aa57b3701" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw", "scenarioId": "interface-contract", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 39113 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "interface-contract", "bytes": 932, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.027645555555555555 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 7 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 39113 }, { "source": "facts", "path": "outcome.durationMs", "value": 49762 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "interface-contract", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj", "runDir": "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj", "finishedAt": "2026-10-28T11:56:56.949Z", "rawSuccess": true, "rawReason": "tsc emit clean + node integration.mjs passed: 5 event(s) across 3 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "durationMs": 52656, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 126.37, "nativePrefillTokensPerSec": 1694.57, "inputTokens": 15656, "outputTokens": 1786, "peakRssMb": 17643, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 52656, "derived": { "durationMs": 40158, "meanTokensPerSec": 126.37, "genTokensPerSec": 126.37, "genTokens": 1786, "promptTokensPerSec": 1694.57, "peakRssMb": 17643, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7324, "output": 23 } } }, "usage": { "available": true, "totalInputTokens": 15656, "totalOutputTokens": 1786, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7324, "todayTokensCached": 7156, "todayTokensOut": 23, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7324, "totalTokensCached": 7156, "totalTokensOut": 23, "totalCost": 0, "medianOutputTokensPerSec": 92.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 92.4, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7324, "cachedInputTokens": 7156, "outputTokens": 23, "cost": 0, "durationMs": 17520, "outputTokensPerSec": 92.35354616627835, "at": "2026-10-28T11:56:27.408Z" }, "lastUpdated": "2026-10-28T11:56:27.408Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:56:47.709Z" } }, "source": "engine-log" }, "process": { "sampleCount": 8, "peakRssMb": 17643, "peakCpuPercent": 268.4, "firstAtMs": 0, "lastAtMs": 38796 }, "gpu": { "available": true, "sampleCount": 8, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 27726, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:56:04.293Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T11:59:49.453Z", "suiteId": "developer", "scenarioId": "interface-contract", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 27726, "totalMb": 31865 }, "evidenceHashes": { "result.json": "a4d2a96f28ef1ac7c9bee74b0dcd0f8f063e6a0e7f391e904632fb4b4f7595f4", "score.json": "44c096e0b88f28d0c95466fae6bf1ca354b73fdff4cb9421be7f0518dd030f94", "metrics.json": "711b914340390393dcf55a825100ea3040b649996e3026e9901e81632a4765d5", "codex-inline-judge.json": "f9ce2fd264b1b8b47c2675fc7d2ce9971c598e6c76b4c2ab2c3cbe421c81d9ee" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj", "scenarioId": "interface-contract", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34086 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "interface-contract", "bytes": 1355, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.029253333333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 11 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34086 }, { "source": "facts", "path": "outcome.durationMs", "value": 52656 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "craftbook-code-review", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs", "runDir": "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs", "finishedAt": "2026-10-28T11:58:55.182Z", "rawSuccess": true, "rawReason": "craftbook-code-review passed 4 deterministic craftbook checks", "durationMs": 114695, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 7, "judgeArtifactCandidates": [ "artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 117.15, "nativePrefillTokensPerSec": 1892.79, "inputTokens": 70707, "outputTokens": 3186, "peakRssMb": 19920, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 114695, "derived": { "durationMs": 107148, "meanTokensPerSec": 117.15, "genTokensPerSec": 117.15, "genTokens": 3186, "promptTokensPerSec": 1892.79, "peakRssMb": 19920, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35420, "output": 531 } } }, "usage": { "available": true, "totalInputTokens": 70707, "totalOutputTokens": 3186, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 35420, "todayTokensCached": 34719, "todayTokensOut": 531, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 35420, "totalTokensCached": 34719, "totalTokensOut": 531, "totalCost": 0, "medianOutputTokensPerSec": 117.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 117.5, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12835, "cachedInputTokens": 12608, "outputTokens": 33, "cost": 0, "durationMs": 27594, "outputTokensPerSec": 152.0702944936297, "at": "2026-10-28T11:58:37.782Z" }, "lastUpdated": "2026-10-28T11:58:37.782Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T11:58:51.216Z" } }, "source": "engine-log" }, "process": { "sampleCount": 20, "peakRssMb": 19920, "peakCpuPercent": 274.8, "firstAtMs": 0, "lastAtMs": 106055 }, "gpu": { "available": true, "sampleCount": 20, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 20, "peakUsedMb": 29835, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:57:00.487Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:05:15.438Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Complete report, local brief, payment source, conventions, manifest and entire seven-file unified diff read against the frozen original prompt and exact workflow fidelity/usefulness axes. Combined input is below 20 KiB. Captured terminal result separately confirms four craftbook gates passed. Source review only; no fixture execution or external service calls.", "suiteId": "developer", "scenarioId": "craftbook-code-review", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs", "artifactPath": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "contextArtifactPaths": [ "workspace/code-review-workflow-eval/source/craftbook-eval-brief.md", "workspace/code-review-workflow-eval/src/payment.js", "workspace/code-review-workflow-eval/docs/review-conventions.md", "artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff" ], "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 8, "usefulness": 6 }, "justification": "Follows the required report structure, ranked findings and request-changes verdict. SQL injection, pagination and swallowed errors are actionable and correctly located; no deliberate safe pattern is falsely flagged. It misses export's error-path file-handle leak, understates settings failure severity, and uses unmapped [3] citations.", "artifactSha256": "46f83785a6d4f1f42e1285c47cbb7fcc23131f0d55ceb16b91f11cfa2193f277", "artifactBytes": 1831, "contextArtifacts": [ { "path": "workspace/code-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 1372, "sha256": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16" }, { "path": "workspace/code-review-workflow-eval/src/payment.js", "bytes": 596, "sha256": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee" }, { "path": "workspace/code-review-workflow-eval/docs/review-conventions.md", "bytes": 1389, "sha256": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b" }, { "path": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "bytes": 1113, "sha256": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e" }, { "path": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff", "bytes": 4396, "sha256": "a1744bbef7b2c7cbd7ab3feba3df59a32a472f32781773715a67b8525758b728" } ], "inputBytes": 10697, "inputLimitBytes": 20480, "truncated": false, "meanScore": 7 }, "systemMemory": { "sampleCount": 20, "peakUsedMb": 29835, "totalMb": 31865 }, "evidenceHashes": { "result.json": "a45e22ee2632f7f19cf15c9f963934b199823c0acee0f1271e83cdaadb357177", "score.json": "bd90a7903169451e3c9010df0540d859e651d1fb2149fc5cbf5f96a572029ea2", "metrics.json": "4b711f863ead8649aedaf1a0dae995f8c1f88414337564da3e0a2fc7d9058634", "codex-inline-judge.json": "90fb28bcea7f56522195be44f0c27e469b54f0bf77a64c0c924ba9ef8420d4ff" }, "rubricId": "craftbook-code-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs", "scenarioId": "craftbook-code-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 66599 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-code-review", "bytes": 1829, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06371944444444444 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 18 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 66599 }, { "source": "facts", "path": "outcome.durationMs", "value": 114695 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "craftbook-code-review", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b", "runDir": "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b", "finishedAt": "2026-10-28T12:00:40.437Z", "rawSuccess": true, "rawReason": "craftbook-code-review passed 4 deterministic craftbook checks", "durationMs": 104137, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 7.5, "judgeArtifactCandidates": [ "artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 124.16, "nativePrefillTokensPerSec": 1937.95, "inputTokens": 67715, "outputTokens": 2936, "peakRssMb": 17252, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 104137, "derived": { "durationMs": 95445, "meanTokensPerSec": 124.16, "genTokensPerSec": 124.16, "genTokens": 2936, "promptTokensPerSec": 1937.95, "peakRssMb": 17252, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 33273, "output": 641 } } }, "usage": { "available": true, "totalInputTokens": 67715, "totalOutputTokens": 2936, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 33273, "todayTokensCached": 21621, "todayTokensOut": 641, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 33273, "totalTokensCached": 21621, "totalTokensOut": 641, "totalCost": 0, "medianOutputTokensPerSec": 132, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 132, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12123, "cachedInputTokens": 11909, "outputTokens": 33, "cost": 0, "durationMs": 18754, "outputTokensPerSec": 150.8587160980393, "at": "2026-10-28T12:00:14.390Z" }, "lastUpdated": "2026-10-28T12:00:14.390Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:00:34.890Z" } }, "source": "engine-log" }, "process": { "sampleCount": 18, "peakRssMb": 17252, "peakCpuPercent": 278, "firstAtMs": 1, "lastAtMs": 94369 }, "gpu": { "available": true, "sampleCount": 18, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 27385, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T11:58:56.300Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:05:15.444Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Independently read the complete second report, manifest and seven-file diff. Its own brief, payment source and conventions were hash-verified identical to the fully read context. Exact frozen workflow fidelity/usefulness axes, combined input below 20 KiB. Captured terminal result separately confirms four craftbook gates passed. No code execution or external validation.", "suiteId": "developer", "scenarioId": "craftbook-code-review", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b", "artifactPath": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "contextArtifactPaths": [ "workspace/code-review-workflow-eval/source/craftbook-eval-brief.md", "workspace/code-review-workflow-eval/src/payment.js", "workspace/code-review-workflow-eval/docs/review-conventions.md", "artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff" ], "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 8, "usefulness": 7 }, "justification": "Structured, actionable review catches both injection sites, leaked handle, swallowed settings error, pagination and debug logging without false positives on sanctioned patterns. However pagination cites closing-brace line 7 instead of calculation line 5, and settings cites line 8 instead of return line 7. Severity ranking understates pagination/logging.", "artifactSha256": "c114a51e8c04871167eed0104627d6808b4845425048a41b0da60a6475a56b4e", "artifactBytes": 2279, "contextArtifacts": [ { "path": "workspace/code-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 1372, "sha256": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16" }, { "path": "workspace/code-review-workflow-eval/src/payment.js", "bytes": 596, "sha256": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee" }, { "path": "workspace/code-review-workflow-eval/docs/review-conventions.md", "bytes": 1389, "sha256": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b" }, { "path": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "bytes": 1113, "sha256": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e" }, { "path": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff", "bytes": 4396, "sha256": "a1744bbef7b2c7cbd7ab3feba3df59a32a472f32781773715a67b8525758b728" } ], "inputBytes": 11145, "inputLimitBytes": 20480, "truncated": false, "meanScore": 7.5 }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 27385, "totalMb": 31865 }, "evidenceHashes": { "result.json": "1bc884025e7eb5097f5ed97e157a12b42f457d9ccac212dde1560ebcf4e2b1dd", "score.json": "e7ad990ac8ac6862095bcc36b43993d9e4d3e847004b65e0ce391b2024d0fa47", "metrics.json": "bd9dcee0760114b5c960b941e5523b7d9a477c9908aaf5bfd5c1eecd5382c6be", "codex-inline-judge.json": "56ce0ea6bf5fef632190bfadfe5fdcf42d2c4f4d2ad24732f3f2469c2358ac2d" }, "rubricId": "craftbook-code-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b", "scenarioId": "craftbook-code-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 57447 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-code-review", "bytes": 2277, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.05785388888888889 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 14 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 57447 }, { "source": "facts", "path": "outcome.durationMs", "value": 104137 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "craftbook-code-review", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6", "runDir": "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6", "finishedAt": "2026-10-28T12:02:57.581Z", "rawSuccess": true, "rawReason": "craftbook-code-review passed 4 deterministic craftbook checks", "durationMs": 135813, "rawFixedComposite": 9.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 7, "judgeArtifactCandidates": [ "artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 120.75, "nativePrefillTokensPerSec": 1947.13, "inputTokens": 82617, "outputTokens": 5814, "peakRssMb": 14100, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 135813, "derived": { "durationMs": 128729, "meanTokensPerSec": 120.75, "genTokensPerSec": 120.75, "genTokens": 5814, "promptTokensPerSec": 1947.13, "peakRssMb": 14100, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35801, "output": 567 } } }, "usage": { "available": true, "totalInputTokens": 82617, "totalOutputTokens": 5814, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 35801, "todayTokensCached": 23249, "todayTokensOut": 567, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 35801, "totalTokensCached": 23249, "totalTokensOut": 567, "totalCost": 0, "medianOutputTokensPerSec": 131.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 131.7, "turns": 3 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12385, "cachedInputTokens": 12187, "outputTokens": 33, "cost": 0, "durationMs": 17388, "outputTokensPerSec": 154.35374018406682, "at": "2026-10-28T12:02:38.676Z" }, "lastUpdated": "2026-10-28T12:02:38.676Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:02:53.770Z" } }, "source": "engine-log" }, "process": { "sampleCount": 24, "peakRssMb": 14100, "peakCpuPercent": 265.8, "firstAtMs": 1, "lastAtMs": 127670 }, "gpu": { "available": true, "sampleCount": 24, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 24, "peakUsedMb": 24324, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:00:41.768Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:05:15.447Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Independently read the complete third report, manifest and seven-file diff. Its own brief, payment source and conventions were hash-verified identical to the fully read context. Exact frozen workflow fidelity/usefulness axes, combined input below 20 KiB. Captured terminal result separately confirms four craftbook gates passed. No code execution or external validation.", "suiteId": "developer", "scenarioId": "craftbook-code-review", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6", "artifactPath": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "contextArtifactPaths": [ "workspace/code-review-workflow-eval/source/craftbook-eval-brief.md", "workspace/code-review-workflow-eval/src/payment.js", "workspace/code-review-workflow-eval/docs/review-conventions.md", "artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff" ], "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 8, "usefulness": 6 }, "justification": "Meets the required structure and gives a supported request-changes verdict with useful injection, leak, settings and pagination fixes. No safe-pattern false positives. It omits forbidden payment debug logging, mislocates pagination/settings findings at closing braces, and leaves [3] references unmapped. Pagination severity is understated.", "artifactSha256": "ddb1f84d7675762fe84648fe1bcd6401039bd791f68fda1ccebcdde368513098", "artifactBytes": 1976, "contextArtifacts": [ { "path": "workspace/code-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 1372, "sha256": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16" }, { "path": "workspace/code-review-workflow-eval/src/payment.js", "bytes": 596, "sha256": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee" }, { "path": "workspace/code-review-workflow-eval/docs/review-conventions.md", "bytes": 1389, "sha256": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b" }, { "path": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "bytes": 1113, "sha256": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e" }, { "path": "artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff", "bytes": 4396, "sha256": "a1744bbef7b2c7cbd7ab3feba3df59a32a472f32781773715a67b8525758b728" } ], "inputBytes": 10842, "inputLimitBytes": 20480, "truncated": false, "meanScore": 7 }, "systemMemory": { "sampleCount": 24, "peakUsedMb": 24324, "totalMb": 31865 }, "evidenceHashes": { "result.json": "cee29fa25016977aa1e7de0955ed9fc542ef81f5e2cc0753422259ff93987cff", "score.json": "55b11eec9c0ef99a3742258dec29f6e6edce3d2dbb50d853846703598078674c", "metrics.json": "1950815fb942385e784c1ff91f286c3b5ab4e62a72cc27309601ce0b084ea824", "codex-inline-judge.json": "0760af5dabd641dd4de76a2064bf04a41523d8046e80569526e7f869d684731b" }, "rubricId": "craftbook-code-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6", "scenarioId": "craftbook-code-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 95428 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-code-review", "bytes": 1974, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07545166666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 17 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 95428 }, { "source": "facts", "path": "outcome.durationMs", "value": 135813 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke", "runDir": "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke", "finishedAt": "2026-10-28T12:07:37.643Z", "rawSuccess": true, "rawReason": "craftbook-api-contract-review passed 6 deterministic craftbook checks", "durationMs": 275717, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 5, "judgeArtifactCandidates": [ "artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 117.16, "nativePrefillTokensPerSec": 1903.95, "inputTokens": 143698, "outputTokens": 15360, "peakRssMb": 18124, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 275717, "derived": { "durationMs": 268653, "meanTokensPerSec": 117.16, "genTokensPerSec": 117.16, "genTokens": 15360, "promptTokensPerSec": 1903.95, "peakRssMb": 18124, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 80245, "output": 2998 } } }, "usage": { "available": true, "totalInputTokens": 143698, "totalOutputTokens": 15360, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 80245, "todayTokensCached": 77972, "todayTokensOut": 2998, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 80245, "totalTokensCached": 77972, "totalTokensOut": 2998, "totalCost": 0, "medianOutputTokensPerSec": 126.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 126.9, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16361, "cachedInputTokens": 16304, "outputTokens": 199, "cost": 0, "durationMs": 45123, "outputTokensPerSec": 103.38874900658033, "at": "2026-10-28T12:07:31.337Z" }, "lastUpdated": "2026-10-28T12:07:31.337Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:07:33.538Z" } }, "source": "engine-log" }, "process": { "sampleCount": 49, "peakRssMb": 18124, "peakCpuPercent": 264.8, "firstAtMs": 18, "lastAtMs": 267633 }, "gpu": { "available": true, "sampleCount": 49, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 28907, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:03:01.926Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:09:35.794Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read the complete 2400-byte API report, scope, endpoint review, local brief, OpenAPI contract, both route files and contract conventions against the original prompt and exact workflow fidelity/usefulness axes. Combined 16072-byte input is below the 20 KiB cap with no clipping. Captured result separately confirms six deterministic workflow gates. Assessment uses saved local fixtures only; no endpoint execution or external validation.", "suiteId": "developer", "scenarioId": "craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke", "artifactPath": "artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "contextArtifactPaths": [ "artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md", "workspace/api-contract-review-workflow-eval/openapi.yaml", "workspace/api-contract-review-workflow-eval/src/routes.ts", "workspace/api-contract-review-workflow-eval/src/session-routes.ts", "workspace/api-contract-review-workflow-eval/docs/api-contract.md" ], "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 7, "usefulness": 3 }, "justification": "Produces the required scope, review, snippets and verdict, but falsely claims three schemas lack IDs already present and flags explicitly valid optional-null email behavior. It misses absent history implementation and undocumented widget lookup. retryCount correction is useful; backward-compatible yes is unsupported for the proposed path rename.", "artifactSha256": "6b9b071b48e5b0b52fb454231f77757850957f2fd10dfb2e46631053f7f04942", "artifactBytes": 2400, "contextArtifacts": [ { "path": "artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "bytes": 2147, "sha256": "5b7ab88aa9ca328a73b0d0206f5f09f4e1c2b1866f8b3642677fe672c26c7686" }, { "path": "artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "bytes": 3337, "sha256": "2bef943d3844b0d8adc402c01b763750ce319462b2fc4cdad69963224cd20629" }, { "path": "workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 2833, "sha256": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97" }, { "path": "workspace/api-contract-review-workflow-eval/openapi.yaml", "bytes": 3167, "sha256": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a" }, { "path": "workspace/api-contract-review-workflow-eval/src/routes.ts", "bytes": 1146, "sha256": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453" }, { "path": "workspace/api-contract-review-workflow-eval/src/session-routes.ts", "bytes": 176, "sha256": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8" }, { "path": "workspace/api-contract-review-workflow-eval/docs/api-contract.md", "bytes": 866, "sha256": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1" } ], "inputBytes": 16072, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5 }, "systemMemory": { "sampleCount": 49, "peakUsedMb": 28907, "totalMb": 31865 }, "evidenceHashes": { "result.json": "127d54509e433560c20382e64ca1293679996e3c8a4469220442228922b912c8", "score.json": "a21f26f7ea57a4af4765d9ad02edad1286572f99c365d20f692b45477754f8e9", "metrics.json": "75654adf9d62ee18179a545864f1e1071eadcb038f7b3ebb7467b2e913929641", "codex-inline-judge.json": "7195bca94eab794679a0b9a355ba51e6138ff40878bb1ceedc176788b76b8863" }, "rubricId": "craftbook-api-contract-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke", "scenarioId": "craftbook-api-contract-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 46815 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-api-contract-review", "bytes": 2346, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.15317611111111112 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 34 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 46815 }, { "source": "facts", "path": "outcome.durationMs", "value": 275717 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm", "runDir": "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm", "finishedAt": "2026-10-28T12:12:40.652Z", "rawSuccess": true, "rawReason": "craftbook-api-contract-review passed 6 deterministic craftbook checks", "durationMs": 302320, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 5.5, "judgeArtifactCandidates": [ "artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 121.8, "nativePrefillTokensPerSec": 1924.91, "inputTokens": 160845, "outputTokens": 15463, "peakRssMb": 18665, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 302320, "derived": { "durationMs": 295133, "meanTokensPerSec": 121.8, "genTokensPerSec": 121.8, "genTokens": 15463, "promptTokensPerSec": 1924.91, "peakRssMb": 18665, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 106896, "output": 5697 } } }, "usage": { "available": true, "totalInputTokens": 160845, "totalOutputTokens": 15463, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 106896, "todayTokensCached": 77007, "todayTokensOut": 5697, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 106896, "totalTokensCached": 77007, "totalTokensOut": 5697, "totalCost": 0, "medianOutputTokensPerSec": 137, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 137, "turns": 9 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7989, "cachedInputTokens": 7818, "outputTokens": 92, "cost": 0, "durationMs": 12150, "outputTokensPerSec": 140.57310573878118, "at": "2026-10-28T12:12:19.981Z" }, "lastUpdated": "2026-10-28T12:12:19.981Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:12:36.746Z" } }, "source": "engine-log" }, "process": { "sampleCount": 54, "peakRssMb": 18665, "peakCpuPercent": 263.9, "firstAtMs": 0, "lastAtMs": 294062 }, "gpu": { "available": true, "sampleCount": 54, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 54, "peakUsedMb": 28937, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:07:38.332Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:19:19.711Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Independently read the complete second API report (2417 bytes), review and scope. All five of its seeded inputs were hash-verified identical to the fully read first-trial fixtures. Judged against the original prompt and exact workflow fidelity/usefulness axes. Combined 15933 bytes, no clipping. Captured result separately confirms six deterministic gates. Saved-source review only, no endpoint execution or external validation.", "suiteId": "developer", "scenarioId": "craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm", "artifactPath": "artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "contextArtifactPaths": [ "artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md", "workspace/api-contract-review-workflow-eval/openapi.yaml", "workspace/api-contract-review-workflow-eval/src/routes.ts", "workspace/api-contract-review-workflow-eval/src/session-routes.ts", "workspace/api-contract-review-workflow-eval/docs/api-contract.md" ], "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 7, "usefulness": 4 }, "justification": "Delivers the expected phased artifacts, concrete snippets and verdict, with a useful retryCount correction. It misses the absent history handler and undocumented widget lookup. Auth guidance invents an undeclared scope scheme and supplies a string instead of a scope list. Calling v1.0.0 new does not establish compatibility for the proposed rename.", "artifactSha256": "fb46dc40e4f2649bb536c4c8675ce44b5f8f273a851d557e0f679527a6f7c699", "artifactBytes": 2417, "contextArtifacts": [ { "path": "artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "bytes": 3230, "sha256": "b75ca7119b37bc9480106bcd478089319e5b7b4e9635dd53e64dc9050d755c86" }, { "path": "artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "bytes": 2098, "sha256": "a29d8ccb0ada4093aeb54fdd6888bb72eb9d0c1ee133406e5b244d1bec633fbd" }, { "path": "workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 2833, "sha256": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97" }, { "path": "workspace/api-contract-review-workflow-eval/openapi.yaml", "bytes": 3167, "sha256": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a" }, { "path": "workspace/api-contract-review-workflow-eval/src/routes.ts", "bytes": 1146, "sha256": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453" }, { "path": "workspace/api-contract-review-workflow-eval/src/session-routes.ts", "bytes": 176, "sha256": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8" }, { "path": "workspace/api-contract-review-workflow-eval/docs/api-contract.md", "bytes": 866, "sha256": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1" } ], "inputBytes": 15933, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.5 }, "systemMemory": { "sampleCount": 54, "peakUsedMb": 28937, "totalMb": 31865 }, "evidenceHashes": { "result.json": "c342198b1658549510559915ea82e0a9a17d86b3eb32da91c80995a5aaef4ecb", "score.json": "b8b99e9fe7793d9ac4d6bd5902b393a641f31f339829a6f53a6da4c63a383c01", "metrics.json": "35f8bd229f5a9fe277e6c0b1c0d9bbc39498ac1279a801ae667fcb17298cda44", "codex-inline-judge.json": "20ea8de02e730a2b13235eb03ac5f87e7d2572388b6a1d6c4efaa22e91bb72bc" }, "rubricId": "craftbook-api-contract-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm", "scenarioId": "craftbook-api-contract-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33758 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-api-contract-review", "bytes": 2387, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.16795555555555555 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 30 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33758 }, { "source": "facts", "path": "outcome.durationMs", "value": 302320 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc", "runDir": "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc", "finishedAt": "2026-10-28T12:26:41.786Z", "rawSuccess": true, "rawReason": "craftbook-api-contract-review passed 6 deterministic craftbook checks", "durationMs": 840006, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 4.5, "judgeArtifactCandidates": [ "artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 118.59, "nativePrefillTokensPerSec": 1923.23, "inputTokens": 528225, "outputTokens": 47031, "peakRssMb": 18892, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 840006, "derived": { "durationMs": 835213, "meanTokensPerSec": 118.59, "genTokensPerSec": 118.59, "genTokens": 47031, "promptTokensPerSec": 1923.23, "peakRssMb": 18892, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 451506, "output": 25419 } } }, "usage": { "available": true, "totalInputTokens": 528225, "totalOutputTokens": 47031, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 27, "todayTokensIn": 451506, "todayTokensCached": 256053, "todayTokensOut": 25419, "todayCost": 0, "totalTurns": 27, "totalTokensIn": 451506, "totalTokensCached": 256053, "totalTokensOut": 25419, "totalCost": 0, "medianOutputTokensPerSec": 125.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 125.1, "turns": 27 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 22531, "outputTokens": 1538, "cost": 0, "durationMs": 26147, "outputTokensPerSec": 121.60846299957679, "at": "2026-10-28T12:26:38.039Z" }, "lastUpdated": "2026-10-28T12:26:38.039Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:26:40.242Z" } }, "source": "engine-log" }, "process": { "sampleCount": 151, "peakRssMb": 18892, "peakCpuPercent": 268.4, "firstAtMs": 1, "lastAtMs": 834196 }, "gpu": { "available": true, "sampleCount": 151, "peakUtilPercent": 100, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 151, "peakUsedMb": 29491, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:12:41.780Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:31:46.578Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Independently read the entire third report (3333 bytes), review and scope; hash-verified all five seeded inputs identical to the previously read fixtures. Combined input 18969 bytes, no clipping. Applied exact frozen workflow fidelity/usefulness axes and original prompt. Captured result separately confirms six deterministic gates after repairs. Source-based review only; no live endpoints or external validation.", "suiteId": "developer", "scenarioId": "craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc", "artifactPath": "artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "contextArtifactPaths": [ "artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md", "workspace/api-contract-review-workflow-eval/openapi.yaml", "workspace/api-contract-review-workflow-eval/src/routes.ts", "workspace/api-contract-review-workflow-eval/src/session-routes.ts", "workspace/api-contract-review-workflow-eval/docs/api-contract.md" ], "artifactKind": "markdown", "scoreAxes": { "workflow fidelity": 6, "usefulness": 3 }, "justification": "Produces the phased artifacts and snippets, but scope lacks an endpoint inventory. The final report misses absent history implementation, undocumented widget lookup and numeric retryCount drift. It labels path changes breaking while declaring compatibility yes, and assumes mandatory auth/versioning without fixture support. Several snippets lack integration detail.", "artifactSha256": "0909c447295f69b9d74a01dbdb36eb7c4b034073081811afe7d8f057f7124df1", "artifactBytes": 3333, "contextArtifacts": [ { "path": "artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "bytes": 4606, "sha256": "e5cc777709a85552ac504fdb009dabe6a77390c98dc5c78257bcf9f69259fbe5" }, { "path": "artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "bytes": 2842, "sha256": "36a4585c03a691cd04458a2ac53704241d0c0ede7abbc79a5087dd7e3c079c8c" }, { "path": "workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 2833, "sha256": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97" }, { "path": "workspace/api-contract-review-workflow-eval/openapi.yaml", "bytes": 3167, "sha256": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a" }, { "path": "workspace/api-contract-review-workflow-eval/src/routes.ts", "bytes": 1146, "sha256": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453" }, { "path": "workspace/api-contract-review-workflow-eval/src/session-routes.ts", "bytes": 176, "sha256": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8" }, { "path": "workspace/api-contract-review-workflow-eval/docs/api-contract.md", "bytes": 866, "sha256": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1" } ], "inputBytes": 18969, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4.5 }, "systemMemory": { "sampleCount": 151, "peakUsedMb": 29491, "totalMb": 31865 }, "evidenceHashes": { "result.json": "8ff57592e72a75356c6ab1f1d2346f41e6349b2c364c96587c2cf19cd783b0b3", "score.json": "58fd312323193d10b25d633d21bcc4d0780dd314e02c03ea316896823dbc8938", "metrics.json": "c8f52be1d22494e4db9d38d373c8ca001ddeea34afb191079e9e86d3406b317a", "codex-inline-judge.json": "9aca8a979b9ad7ebafa513d2280b8d4a2273abdce74296811b1d0d4e8600c1db" }, "rubricId": "craftbook-api-contract-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc", "scenarioId": "craftbook-api-contract-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 45250 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-api-contract-review", "bytes": 3285, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.46667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 65 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 45250 }, { "source": "facts", "path": "outcome.durationMs", "value": 840006 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn", "runDir": "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn", "finishedAt": "2026-10-28T12:29:13.226Z", "rawSuccess": false, "rawReason": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"", "durationMs": 147082, "rawFixedComposite": 6.3, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 5, "judgeArtifactCandidates": [ "workspace/security-maintenance-eval/security-review/REPORT.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 134.83, "nativePrefillTokensPerSec": 1858.44, "inputTokens": 49890, "outputTokens": 10612, "peakRssMb": 19404, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 147082, "derived": { "durationMs": 139381, "meanTokensPerSec": 134.83, "genTokensPerSec": 134.83, "genTokens": 10612, "promptTokensPerSec": 1858.44, "peakRssMb": 19404, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 97789, "output": 8430 } } }, "usage": { "available": true, "totalInputTokens": 49890, "totalOutputTokens": 10612, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 97789, "todayTokensCached": 66565, "todayTokensOut": 8430, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 97789, "totalTokensCached": 66565, "totalTokensOut": 8430, "totalCost": 0, "medianOutputTokensPerSec": 137.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 137.4, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15592, "cachedInputTokens": 14312, "outputTokens": 1396, "cost": 0, "durationMs": 11359, "outputTokensPerSec": 137.40946612334605, "at": "2026-10-28T12:29:03.017Z" }, "lastUpdated": "2026-10-28T12:29:03.017Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:29:08.658Z" } }, "source": "engine-log" }, "process": { "sampleCount": 26, "peakRssMb": 19404, "peakCpuPercent": 261.5, "firstAtMs": 0, "lastAtMs": 138357 }, "gpu": { "available": true, "sampleCount": 26, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 29908, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:26:46.144Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:31:46.583Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read the full 3169-byte report, complete findings JSON and all four seeded source/convention files against the original prompt and exact verification/systemic analysis/remediation axes. Combined 7361 bytes, no clipping. Judged despite deterministic failure on machine-readable schema requirements. Static source-to-sink inspection only; no exploit execution, source edits or external calls.", "suiteId": "developer", "scenarioId": "craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn", "artifactPath": "workspace/security-maintenance-eval/security-review/REPORT.md", "contextArtifactPaths": [ "workspace/security-maintenance-eval/security-review/findings.json", "workspace/security-maintenance-eval/src/admin.ts", "workspace/security-maintenance-eval/src/profile.ts", "workspace/security-maintenance-eval/src/reports.ts", "workspace/security-maintenance-eval/docs/security.md" ], "artifactKind": "markdown", "scoreAxes": { "verification": 5, "systemic analysis": 5, "remediation": 5 }, "justification": "Correctly traces both admin injections and absent requireAdmin, with concrete fixes and RCE blast radius. However it records allow-listed ORDER BY and constant logrotate as medium vulnerabilities despite explicit safe exceptions. Root-cause analysis mostly merges injection issues, and JSON lacks theme fields; report severity detail is incomplete.", "artifactSha256": "40d3db3f3726284716508e60d9d88064bdc24cd3f2b9a9d15b6ec620e48a1264", "artifactBytes": 3169, "contextArtifacts": [ { "path": "workspace/security-maintenance-eval/security-review/findings.json", "bytes": 2330, "sha256": "41fb00955c9bb609c29bd70be4ea968953e8c2985b315c0fa4a3254f5fafa3a7" }, { "path": "workspace/security-maintenance-eval/src/admin.ts", "bytes": 378, "sha256": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7" }, { "path": "workspace/security-maintenance-eval/src/profile.ts", "bytes": 223, "sha256": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29" }, { "path": "workspace/security-maintenance-eval/src/reports.ts", "bytes": 613, "sha256": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790" }, { "path": "workspace/security-maintenance-eval/docs/security.md", "bytes": 648, "sha256": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d" } ], "inputBytes": 7361, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5 }, "systemMemory": { "sampleCount": 26, "peakUsedMb": 29908, "totalMb": 31865 }, "evidenceHashes": { "result.json": "43be5c5d39478f9ff53b2f307c7161b1845e2bc2b2211dd1cda56357092f313e", "score.json": "c30d6afe6f1e10bec70afcf1ca92d4a9e2de5452745ef58ad6f6f9a29245f7e0", "metrics.json": "73199d2ad4c3c23984bd65f42d511f53d7b8badd412fd4c88fd232a13a7c4758", "codex-inline-judge.json": "9df5bc348dc2afe180d65cf3a098e3c26850e3a1ff01d8258a91a66978593b1d" }, "rubricId": "craftbook-deep-security-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn", "scenarioId": "craftbook-deep-security-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34080 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 3169, "score": 9, "scoreMax": 11, "signals": [], "failReason": "security-review/findings.json: record 0 is missing required field \"theme\"" } } ] }, "quality": { "score": 8.2, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 81.8% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34080 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 3169, "score": 9, "scoreMax": 11, "signals": [], "failReason": "security-review/findings.json: record 0 is missing required field \"theme\"" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.054474814814814816 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 13 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34080 }, { "source": "facts", "path": "outcome.durationMs", "value": 147082 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6.3, "band": "needs-tuning" } }, { "suiteId": "developer", "scenarioId": "craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri", "runDir": "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri", "finishedAt": "2026-10-28T12:32:09.823Z", "rawSuccess": false, "rawReason": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"", "durationMs": 175311, "rawFixedComposite": 6.3, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 5.333333333333333, "judgeArtifactCandidates": [ "workspace/security-maintenance-eval/security-review/REPORT.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 131.6, "nativePrefillTokensPerSec": 1915.15, "inputTokens": 78898, "outputTokens": 11236, "peakRssMb": 17405, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 175311, "derived": { "durationMs": 167585, "meanTokensPerSec": 131.6, "genTokensPerSec": 131.6, "genTokens": 11236, "promptTokensPerSec": 1915.15, "peakRssMb": 17405, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 102355, "output": 5188 } } }, "usage": { "available": true, "totalInputTokens": 78898, "totalOutputTokens": 11236, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 102355, "todayTokensCached": 69346, "todayTokensOut": 5188, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 102355, "totalTokensCached": 69346, "totalTokensOut": 5188, "totalCost": 0, "medianOutputTokensPerSec": 122.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 122.1, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15929, "cachedInputTokens": 14795, "outputTokens": 868, "cost": 0, "durationMs": 8753, "outputTokensPerSec": 113.99650989231996, "at": "2026-10-28T12:32:02.378Z" }, "lastUpdated": "2026-10-28T12:32:02.378Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:32:05.018Z" } }, "source": "engine-log" }, "process": { "sampleCount": 31, "peakRssMb": 17405, "peakCpuPercent": 267.8, "firstAtMs": 1, "lastAtMs": 166558 }, "gpu": { "available": true, "sampleCount": 31, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 31, "peakUsedMb": 27893, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:29:14.512Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:32:58.274Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Independently read the full second report (3133 bytes) and findings JSON; hash-verified all four seeded source/convention files identical to the fully read first-trial fixtures. Combined 6727 bytes, no clipping. Applied the exact frozen verification/systemic analysis/remediation rubric despite deterministic failure. Static source inspection only; no exploit execution or external calls.", "suiteId": "developer", "scenarioId": "craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri", "artifactPath": "workspace/security-maintenance-eval/security-review/REPORT.md", "contextArtifactPaths": [ "workspace/security-maintenance-eval/security-review/findings.json", "workspace/security-maintenance-eval/src/admin.ts", "workspace/security-maintenance-eval/src/profile.ts", "workspace/security-maintenance-eval/src/reports.ts", "workspace/security-maintenance-eval/docs/security.md" ], "artifactKind": "markdown", "scoreAxes": { "verification": 4, "systemic analysis": 7, "remediation": 5 }, "justification": "Finds the three admin defects and separates unsafe sinks from missing authorization with clear blast radii. Verification contradicts itself: proven admin shell flow is called not statically verifiable, while safe constant logrotate is a high-severity JSON injection despite a safe report status. Fixes are concrete, but themes and cross-artifact consistency remain incomplete.", "artifactSha256": "53207a87fd02a05a9558bfb563f66d9d46411cdbd5cc18f451026b58a9fed967", "artifactBytes": 3133, "contextArtifacts": [ { "path": "workspace/security-maintenance-eval/security-review/findings.json", "bytes": 1732, "sha256": "a6522b60268a47f85190b5eb148d8028b4bbe1f72d4df2e89880bcde3035b75a" }, { "path": "workspace/security-maintenance-eval/src/admin.ts", "bytes": 378, "sha256": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7" }, { "path": "workspace/security-maintenance-eval/src/profile.ts", "bytes": 223, "sha256": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29" }, { "path": "workspace/security-maintenance-eval/src/reports.ts", "bytes": 613, "sha256": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790" }, { "path": "workspace/security-maintenance-eval/docs/security.md", "bytes": 648, "sha256": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d" } ], "inputBytes": 6727, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.333333333333333 }, "systemMemory": { "sampleCount": 31, "peakUsedMb": 27893, "totalMb": 31865 }, "evidenceHashes": { "result.json": "77bb6cde8426e8b17a5a001a1dcfdcfbfa4c3cc6ef3c4b5e255598123f185b50", "score.json": "0d9b2a3672f1e2b7241fa87d9b00422e63393da3349afbde99265091ad032393", "metrics.json": "8d9bb7feca93765898d0d2e3ac8399764dac59532fabea0478efbacbb0cc7f80", "codex-inline-judge.json": "b9fb2c4f7c55f67966346ada33fae4668e941342d147587bfaecca173cfa3018" }, "rubricId": "craftbook-deep-security-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri", "scenarioId": "craftbook-deep-security-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33823 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 3133, "score": 9, "scoreMax": 11, "signals": [], "failReason": "security-review/findings.json: record 0 is missing required field \"theme\"" } } ] }, "quality": { "score": 8.2, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 81.8% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33823 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 3133, "score": 9, "scoreMax": 11, "signals": [], "failReason": "security-review/findings.json: record 0 is missing required field \"theme\"" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06493 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 15 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 33823 }, { "source": "facts", "path": "outcome.durationMs", "value": 175311 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6.3, "band": "needs-tuning" } }, { "suiteId": "developer", "scenarioId": "craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4", "runDir": "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4", "finishedAt": "2026-10-28T12:35:22.814Z", "rawSuccess": false, "rawReason": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"", "durationMs": 192019, "rawFixedComposite": 6.3, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 5, "judgeArtifactCandidates": [ "workspace/security-maintenance-eval/security-review/REPORT.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 132.51, "nativePrefillTokensPerSec": 1916.29, "inputTokens": 82451, "outputTokens": 13730, "peakRssMb": 17441, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10606, "gpuMemoryTotalMb": 12227, "durationMs": 192019, "derived": { "durationMs": 183982, "meanTokensPerSec": 132.51, "genTokensPerSec": 132.51, "genTokens": 13730, "promptTokensPerSec": 1916.29, "peakRssMb": 17441, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 107242, "output": 7225 } } }, "usage": { "available": true, "totalInputTokens": 82451, "totalOutputTokens": 13730, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 107242, "todayTokensCached": 72214, "todayTokensOut": 7225, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 107242, "totalTokensCached": 72214, "totalTokensOut": 7225, "totalCost": 0, "medianOutputTokensPerSec": 138.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 138.2, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 17065, "cachedInputTokens": 15808, "outputTokens": 924, "cost": 0, "durationMs": 7863, "outputTokensPerSec": 138.24173882046696, "at": "2026-10-28T12:35:14.514Z" }, "lastUpdated": "2026-10-28T12:35:14.514Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:35:18.061Z" } }, "source": "engine-log" }, "process": { "sampleCount": 34, "peakRssMb": 17441, "peakCpuPercent": 265.7, "firstAtMs": 1, "lastAtMs": 182973 }, "gpu": { "available": true, "sampleCount": 34, "peakUtilPercent": 99, "peakMemUsedMb": 10606, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 27951, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:32:10.795Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T12:40:38.954Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Independently read the complete third report (3280 bytes) and JSON; hash-verified all four source/convention inputs identical to previously fully read fixtures. Combined 7478 bytes, no clipping. Applied exact frozen verification/systemic analysis/remediation axes despite deterministic failure. Static artifact/source review only, without exploit execution or external calls.", "suiteId": "developer", "scenarioId": "craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4", "artifactPath": "workspace/security-maintenance-eval/security-review/REPORT.md", "contextArtifactPaths": [ "workspace/security-maintenance-eval/security-review/findings.json", "workspace/security-maintenance-eval/src/admin.ts", "workspace/security-maintenance-eval/src/profile.ts", "workspace/security-maintenance-eval/src/reports.ts", "workspace/security-maintenance-eval/docs/security.md" ], "artifactKind": "markdown", "scoreAxes": { "verification": 5, "systemic analysis": 5, "remediation": 5 }, "justification": "Correctly traces the three admin defects and provides concrete safe-execution, SQL and middleware fixes. But safe reports routes remain vulnerability records: speculative audit need and allow-listed SQL get severity labels. The dynamic-identifier parameter suggestion conflicts with local conventions. Systemic analysis is broad; JSON themes remain absent.", "artifactSha256": "6c6c0238ba9cb247668122313750efe7758b1b56d1e68b3974da04b5d33fc1d2", "artifactBytes": 3280, "contextArtifacts": [ { "path": "workspace/security-maintenance-eval/security-review/findings.json", "bytes": 2336, "sha256": "7ec0e91bf4e7ac73fb1d6925668274872a26283904d629e4fbddbedf76ea1495" }, { "path": "workspace/security-maintenance-eval/src/admin.ts", "bytes": 378, "sha256": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7" }, { "path": "workspace/security-maintenance-eval/src/profile.ts", "bytes": 223, "sha256": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29" }, { "path": "workspace/security-maintenance-eval/src/reports.ts", "bytes": 613, "sha256": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790" }, { "path": "workspace/security-maintenance-eval/docs/security.md", "bytes": 648, "sha256": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d" } ], "inputBytes": 7478, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5 }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 27951, "totalMb": 31865 }, "evidenceHashes": { "result.json": "43d1c35be4102d75a667fa39717bf6fbc057146eef08d79339b08cd37207bd4c", "score.json": "4bb6ca8eda03d68129784744d84b54eb3808eff9d05d0d78c2649e14e289f8ad", "metrics.json": "01fc4da9ccb613ce0257bb2ae367406d1004e7a03c3bcb107b93ec6f3b59288e", "codex-inline-judge.json": "529653676e46443c3ec047c19ff371aa0dea2063c6f64343702e6fea33eb101e" }, "rubricId": "craftbook-deep-security-review", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4", "scenarioId": "craftbook-deep-security-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34161 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 3280, "score": 9, "scoreMax": 11, "signals": [], "failReason": "security-review/findings.json: record 0 is missing required field \"theme\"" } } ] }, "quality": { "score": 8.2, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 81.8% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34161 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 3280, "score": 9, "scoreMax": 11, "signals": [], "failReason": "security-review/findings.json: record 0 is missing required field \"theme\"" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07111814814814815 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 15 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34161 }, { "source": "facts", "path": "outcome.durationMs", "value": 192019 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6.3, "band": "needs-tuning" } }, { "suiteId": "developer", "scenarioId": "large-pr-review", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk", "runDir": "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk", "finishedAt": "2026-10-28T12:48:57.832Z", "rawSuccess": true, "rawReason": "complete large-PR review (coverage-120-of-120, coverage-summary, late-defect-citation, request-changes, no-false-missing-api, source-unchanged)", "durationMs": 810832, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 135.16, "nativePrefillTokensPerSec": 1729.55, "inputTokens": 300446, "outputTokens": 70504, "peakRssMb": 20975, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10730, "gpuMemoryTotalMb": 12227, "durationMs": 810832, "derived": { "durationMs": 803746, "meanTokensPerSec": 135.16, "genTokensPerSec": 135.16, "genTokens": 70504, "promptTokensPerSec": 1729.55, "peakRssMb": 20975, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 321880, "output": 11888 } } }, "usage": { "available": true, "totalInputTokens": 300446, "totalOutputTokens": 70504, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 321880, "todayTokensCached": 223936, "todayTokensOut": 11888, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 321880, "totalTokensCached": 223936, "totalTokensOut": 11888, "totalCost": 0, "medianOutputTokensPerSec": 126.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 126.6, "turns": 15 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 24608, "outputTokens": 3026, "cost": 0, "durationMs": 36094, "outputTokensPerSec": 137.0509587632014, "at": "2026-10-28T12:48:50.555Z" }, "lastUpdated": "2026-10-28T12:48:50.555Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T12:48:54.036Z" } }, "source": "engine-log" }, "process": { "sampleCount": 145, "peakRssMb": 20975, "peakCpuPercent": 279.2, "firstAtMs": 0, "lastAtMs": 802638 }, "gpu": { "available": true, "sampleCount": 145, "peakUtilPercent": 100, "peakMemUsedMb": 10730, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 145, "peakUsedMb": 31353, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 152576, "totalTokens": 152576, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:35:27.000Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T13:00:15.204Z", "suiteId": "developer", "scenarioId": "large-pr-review", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 145, "peakUsedMb": 31353, "totalMb": 31865 }, "evidenceHashes": { "result.json": "35effc2e3cff7083ef852a083bb37c2407a4e26ce3c058c1d38db3649fea4cde", "score.json": "ca5c3d25dd896f0749dacc661771dd2f1681e1a3f8ff355c70ddd259d063d161", "metrics.json": "cf671c1d6c6afc7f7e851f0032a34e81bace54a16d0c32bd34a81ce99a8c414b", "codex-inline-judge.json": "5e2900b178bdc627267a703d09d33ec725508cb2c6014ad28f3974e46e562198" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk", "scenarioId": "large-pr-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 28884 }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2252311111111111 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 58 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 28884 }, { "source": "facts", "path": "outcome.durationMs", "value": 810832 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "large-pr-review", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i", "runDir": "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i", "finishedAt": "2026-10-28T13:11:28.305Z", "rawSuccess": false, "rawReason": "retry loop (fast-path): sniff \"large-pr-review:5:targetpr-review.md:fr1i0sit8:rp0:rf0\" stuck for 8m — 4 re-writes (6 tool calls) without sniff movement. Artifact produced but never reached success.", "durationMs": 1349609, "rawFixedComposite": 3.3, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 134.89, "nativePrefillTokensPerSec": 1593.34, "inputTokens": 353875, "outputTokens": 87095, "peakRssMb": 20057, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10607, "gpuMemoryTotalMb": 12227, "durationMs": 1349609, "derived": { "durationMs": 1340928, "meanTokensPerSec": 134.89, "genTokensPerSec": 134.89, "genTokens": 87095, "promptTokensPerSec": 1593.34, "peakRssMb": 20057, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 526934, "output": 13883 } } }, "usage": { "available": true, "totalInputTokens": 353875, "totalOutputTokens": 87095, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 526934, "todayTokensCached": 428523, "todayTokensOut": 13883, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 526934, "totalTokensCached": 428523, "totalTokensOut": 13883, "totalCost": 0, "medianOutputTokensPerSec": 121.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 121.6, "turns": 15 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 53169, "cachedInputTokens": 48682, "outputTokens": 3894, "cost": 0, "durationMs": 119242, "outputTokensPerSec": 138.99029424313127, "at": "2026-10-28T13:07:59.839Z" }, "lastUpdated": "2026-10-28T13:07:59.839Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T13:11:26.956Z" } }, "source": "engine-log" }, "process": { "sampleCount": 241, "peakRssMb": 20057, "peakCpuPercent": 269.1, "firstAtMs": 0, "lastAtMs": 1339915 }, "gpu": { "available": true, "sampleCount": 241, "peakUtilPercent": 100, "peakMemUsedMb": 10607, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 241, "peakUsedMb": 30685, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 153600, "totalTokens": 153600, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T12:48:58.696Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T13:18:51.298Z", "suiteId": "developer", "scenarioId": "large-pr-review", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 241, "peakUsedMb": 30685, "totalMb": 31865 }, "evidenceHashes": { "result.json": "2724e260bc3b3da6f0492c45a9c2f7d1df4e40be9d441c47afdbf0f8fa96abca", "score.json": "970ba2c5fe9f1f2f220facdeea57f416e8b64e9ab0210f9b6a8d9e6cbfab1d4e", "metrics.json": "43995bac659bd286397261a9d99870a735c9981c7fabb5655655ae4fe85ec8d4", "codex-inline-judge.json": "85fce5df73d8e2c1e37d1f8c9a7322829365ec7114589457442b4524df23a81c" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i", "scenarioId": "large-pr-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "retry loop (fast-path): sniff \"large-pr-review:5:targetpr-review.md:fr1i0sit8:rp0:rf0\" stuck for 8m — 4 re-writes (6 tool calls) without sniff movement. Artifact produced but never reached success." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34904 }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "quality": { "score": 5, "ruleId": "Q_UNGRADED_AUTHORED_OUTPUT", "summary": "An authored output exists without a sniff verdict; quality is capped at the rubric midpoint.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34904 }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.3748913888888889 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 40 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34904 }, { "source": "facts", "path": "outcome.durationMs", "value": 1349609 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.3, "band": "framework-gap" } }, { "suiteId": "developer", "scenarioId": "large-pr-review", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i", "runDir": "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i", "finishedAt": "2026-10-28T13:34:28.635Z", "rawSuccess": true, "rawReason": "complete large-PR review (coverage-120-of-120, coverage-summary, late-defect-citation, request-changes, no-false-missing-api, source-unchanged)", "durationMs": 1379755, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 136.87, "nativePrefillTokensPerSec": 1640.92, "inputTokens": 510488, "outputTokens": 108415, "peakRssMb": 17572, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10607, "gpuMemoryTotalMb": 12227, "durationMs": 1379755, "derived": { "durationMs": 1348750, "meanTokensPerSec": 136.87, "genTokensPerSec": 136.87, "genTokens": 108415, "promptTokensPerSec": 1640.92, "peakRssMb": 17572, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 297638, "output": 18686 } } }, "usage": { "available": true, "totalInputTokens": 510488, "totalOutputTokens": 108415, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 297638, "todayTokensCached": 132516, "todayTokensOut": 18686, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 297638, "totalTokensCached": 132516, "totalTokensOut": 18686, "totalCost": 0, "medianOutputTokensPerSec": 136.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 136.3, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19553, "cachedInputTokens": 17681, "outputTokens": 54, "cost": 0, "durationMs": 21912, "outputTokensPerSec": 125.90778323906089, "at": "2026-10-28T13:34:18.754Z" }, "lastUpdated": "2026-10-28T13:34:18.754Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T13:34:22.149Z" } }, "source": "engine-log" }, "process": { "sampleCount": 244, "peakRssMb": 17572, "peakCpuPercent": 271.7, "firstAtMs": 1, "lastAtMs": 1347655 }, "gpu": { "available": true, "sampleCount": 244, "peakUtilPercent": 100, "peakMemUsedMb": 10607, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 244, "peakUsedMb": 29364, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 153600, "totalTokens": 153600, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T13:11:28.880Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T13:39:51.996Z", "suiteId": "developer", "scenarioId": "large-pr-review", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 244, "peakUsedMb": 29364, "totalMb": 31865 }, "evidenceHashes": { "result.json": "16af2c3a73721499978f057e41a6be09a1d360ae642a351a9ebb6489ea45d42c", "score.json": "b6b271c9250d9d44582be037ba789f2c44bcd39720ef19e9a39512625cb6a0cb", "metrics.json": "9dd88c4ab9360daa6a2e28e76cddf12f07d196830b492e32d097fc39a459012d", "codex-inline-judge.json": "493573c75c1791740c7ffc8164ce682448f739f040c37f9b90fa5f6f0f2cab24" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i", "scenarioId": "large-pr-review", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51739 }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.3832652777777778 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 46 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51739 }, { "source": "facts", "path": "outcome.durationMs", "value": 1379755 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio", "runDir": "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio", "finishedAt": "2026-10-28T13:50:56.833Z", "rawSuccess": false, "rawReason": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 28 while the failing detail kept changing; latest checked file workspace output. Last failure: tests/profile.test.mjs did not pass when run with node: exit=1 Command failed: Files\\nodejs\\node.exe tests/profile.test.mjs /tests/profile.test.mjs:3 import { loadUser } from '../src/index.js'; ^^^^^^^^ SyntaxError: Identifier 'loadUser' has already been declared at compileSourceTextModule (node:internal/modules/esm/utils:319:16) at ModuleLoader.moduleStrategy (node:internal/modules/esm/translators:89:18) at #translate (node:internal/modules/esm/loader:438:20) at afterLoad (node:internal/modules/esm/loader:506:29) at ModuleLoader.loadAndTranslate (node:internal/modules/esm/loader:511:12) at #getOrCreateModuleJobAfterResolve (node:internal/modules/esm/loader:564:36) at aft", "durationMs": 984659, "rawFixedComposite": 5.4, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 3.3333333333333335, "judgeArtifactCandidates": [ "artifacts/profile-cards/tasks/1/verification.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 121.9, "nativePrefillTokensPerSec": 1442.07, "inputTokens": 292465, "outputTokens": 47854, "peakRssMb": 20586, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10600, "gpuMemoryTotalMb": 12227, "durationMs": 984659, "derived": { "durationMs": 979359, "meanTokensPerSec": 121.9, "genTokensPerSec": 121.9, "genTokens": 47854, "promptTokensPerSec": 1442.07, "peakRssMb": 20586, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 479927, "output": 9747 } } }, "usage": { "available": true, "totalInputTokens": 292465, "totalOutputTokens": 47854, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 24, "todayTokensIn": 479927, "todayTokensCached": 472836, "todayTokensOut": 9747, "todayCost": 0, "totalTurns": 24, "totalTokensIn": 479927, "totalTokensCached": 472836, "totalTokensOut": 9747, "totalCost": 0, "medianOutputTokensPerSec": 141.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 141.4, "turns": 24 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 31336, "cachedInputTokens": 31093, "outputTokens": 234, "cost": 0, "durationMs": 13581, "outputTokensPerSec": 141.78053225507108, "at": "2026-10-28T13:47:05.589Z" }, "lastUpdated": "2026-10-28T13:47:05.589Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T13:50:55.125Z" } }, "source": "engine-log" }, "process": { "sampleCount": 177, "peakRssMb": 20586, "peakCpuPercent": 269.8, "firstAtMs": 3, "lastAtMs": 978011 }, "gpu": { "available": true, "sampleCount": 177, "peakUtilPercent": 100, "peakMemUsedMb": 10600, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 177, "peakUsedMb": 31470, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] profile-cards/1 paused for help (step_stalled) at step \"apply\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/command-tool-availability.json preserves all 16 model-facing rosters with exact daemon line numbers, the first required-run gate note and SHA256 hashes of both source logs.", "Required gate on profile-cards/1, step verify, at 10/28/2026 13:37:39 explicitly demands run_package_script; zero captured model-facing rosters include it.", "Recorded tool calls contain no package runner; the broad registered bridge roster is not the model-visible filtered tool list." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; the underlying policy layer causing omission is not established. Preserve original model-default result and fixed score, strict planned attempt count, artifact flaws and actual judgments/performance. This supplemental eligibility supersedes earlier provisional no-exclusion notes for this trial. No source, dependency, tool permission, gate or model tuning was changed.", "reviewedAt": "2026-10-28T15:44:56.4119865Z" }, "startedAt": "2026-10-28T13:34:32.174Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T14:01:00.093Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read complete verification, enumeration/change/review artifacts, final source/test/history files and rename brief. Examined captured tool chronology and froze the actual pre-edit grep, first successful source write, all test-file line edits, complete per-tool counts and terminal test error in judge-context-codemod.json with recording SHA256. Exact frozen Enumeration completeness/Invariant discipline/Reconciled verification axes; combined selected input below 20 KiB. No new test execution or source changes.", "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio", "artifactPath": "artifacts/profile-cards/tasks/1/verification.md", "contextArtifactPaths": [ "artifacts/profile-cards/tasks/1/sites.md", "artifacts/profile-cards/tasks/1/change-notes.md", "artifacts/profile-cards/tasks/1/review.md", "workspace/profile-cards/source/rename-brief.md", "workspace/profile-cards/package.json", "workspace/profile-cards/src/api.js", "workspace/profile-cards/src/index.js", "workspace/profile-cards/src/profile.js", "workspace/profile-cards/tests/profile.test.mjs", "workspace/profile-cards/CHANGELOG.md", "judge-context-codemod.json" ], "artifactKind": "markdown", "scoreAxes": { "Enumeration completeness": 7, "Invariant discipline": 2, "Reconciled verification": 1 }, "justification": "A real pre-edit search found all seven live sites and justified history exclusions. But test-line edits removed the assertion import and created duplicate loadUser imports, breaking the suite. Verification falsely claims green tests and a residual rescan; the trace has only the initial search, while the captured test run fails. Counts omit the extra mutation.", "artifactSha256": "e0ad9b828fef7ba781794e1028edd24ba3919d387eee6bf02c18984bbfad6990", "artifactBytes": 799, "contextArtifacts": [ { "path": "artifacts/profile-cards/tasks/1/sites.md", "bytes": 747, "sha256": "86764c72ff70c1c28847619902513b9d291ff8dd5199b4696e08a807a72681ad" }, { "path": "artifacts/profile-cards/tasks/1/change-notes.md", "bytes": 558, "sha256": "dacd6451e1d0cf7898e690468a5b28e22441dafa9570b2524d669b917735724f" }, { "path": "artifacts/profile-cards/tasks/1/review.md", "bytes": 1002, "sha256": "e6406c1bbc6e490165a2a2a2beb252ecaeea1ad167bf9068434d522de1a2b39a" }, { "path": "workspace/profile-cards/source/rename-brief.md", "bytes": 534, "sha256": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80" }, { "path": "workspace/profile-cards/package.json", "bytes": 137, "sha256": "d69e9390722b6b5ae20c94d274faf88a16fa95d57293a7d2fcf5ecad97c0650b" }, { "path": "workspace/profile-cards/src/api.js", "bytes": 224, "sha256": "78619f2daba1ddc732dca868c5cdbb606b5c0aad0e8aea977227d361556d5316" }, { "path": "workspace/profile-cards/src/index.js", "bytes": 37, "sha256": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c" }, { "path": "workspace/profile-cards/src/profile.js", "bytes": 225, "sha256": "a723dc6e94beda9791e150daf98b27ae01f94968a7ffecabd4e0ad736641f3b0" }, { "path": "workspace/profile-cards/tests/profile.test.mjs", "bytes": 559, "sha256": "cb6f1aa1b2663a90c2452f43bdadba53df012bb4c7b022366d02545d4ead852c" }, { "path": "workspace/profile-cards/CHANGELOG.md", "bytes": 207, "sha256": "51760c8cae86a3fd0b4ff6a0ea029d890ebc0e64045f32b7bed53e2da145e5ea" }, { "path": "judge-context-codemod.json", "bytes": 5760, "sha256": "eac3beffb4f23f2e3dfb11ec741e28307f74f583f9f3b6599f9195a3a2bca458" } ], "inputBytes": 10789, "inputLimitBytes": 20480, "truncated": false, "meanScore": 3.3333333333333335 }, "systemMemory": { "sampleCount": 177, "peakUsedMb": 31470, "totalMb": 31865 }, "evidenceHashes": { "result.json": "342e21434f36904779395371ff11567064455841886c1d4e80197729b00c4780", "score.json": "558964afbf88d47b7262d6453fb7d199cae3a1c707371df72a635504eace7047", "metrics.json": "cc7540c0cbbbb02c424131895c50142766a07a8bbe9f876d2c9dd7af8b1f0bf7", "codex-inline-judge.json": "d034f4eca79b4a4cfb67b9d9c842b871e2fb8b3370f07b5f87b07ca601bf7642" }, "rubricId": "craftbook-codemod-sweep", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio", "scenarioId": "craftbook-codemod-sweep", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 28 while the failing detail kept changing; latest checked file workspace output. Last failure: tests/profile.test.mjs did not pass when run with node: exit=1 Command failed: Files\\nodejs\\node.exe tests/profile.test.mjs /tests/profile.test.mjs:3 import { loadUser } from '../src/index.js'; ^^^^^^^^ SyntaxError: Identifier 'loadUser' has already been declared at compileSourceTextModule (node:internal/modules/esm/utils:319:16) at ModuleLoader.moduleStrategy (node:internal/modules/esm/translators:89:18) at #translate (node:internal/modules/esm/loader:438:20) at afterLoad (node:internal/modules/esm/loader:506:29) at ModuleLoader.loadAndTranslate (node:internal/modules/esm/loader:511:12) at #getOrCreateModuleJobAfterResolve (node:internal/modules/esm/loader:564:36) at aft" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 31647 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 747, "score": 28, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not reached a terminal step" } } ] }, "quality": { "score": 8.8, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 87.5% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 31647 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 747, "score": 28, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not reached a terminal step" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.18234425925925926 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 254 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 31647 }, { "source": "facts", "path": "outcome.durationMs", "value": 984659 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.4, "band": "needs-tuning" } }, { "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb", "runDir": "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb", "finishedAt": "2026-10-28T13:58:56.729Z", "rawSuccess": false, "rawReason": "craftbook-codemod-sweep stale no-write repair loop: tasks/1/verification.md had unchanged content (882 bytes) for 77s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 27/32: tasks/1/verification.md is missing required content: verification sections in order — nothing in its 882 bytes matches /^##\\s+Residual scan[\\s\\S]*^##\\s+Counts reconciled[\\s\\S]*^##\\s+Suite[\\s\\S]*^##\\s+Result/. Add that content.", "durationMs": 479227, "rawFixedComposite": 4.6, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 3.3333333333333335, "judgeArtifactCandidates": [ "workspace/profile-cards/tasks/1/verification.md", "artifacts/profile-cards/tasks/1/verification.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 109.44, "nativePrefillTokensPerSec": 1742.84, "inputTokens": 183558, "outputTokens": 24150, "peakRssMb": 20791, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 479227, "derived": { "durationMs": 471836, "meanTokensPerSec": 109.44, "genTokensPerSec": 109.44, "genTokens": 24150, "promptTokensPerSec": 1742.84, "peakRssMb": 20791, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 315804, "output": 6019 } } }, "usage": { "available": true, "totalInputTokens": 183558, "totalOutputTokens": 24150, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 24, "todayTokensIn": 315804, "todayTokensCached": 286487, "todayTokensOut": 6019, "todayCost": 0, "totalTurns": 24, "totalTokensIn": 315804, "totalTokensCached": 286487, "totalTokensOut": 6019, "totalCost": 0, "medianOutputTokensPerSec": 116.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 116.8, "turns": 24 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9030, "cachedInputTokens": 8118, "outputTokens": 95, "cost": 0, "durationMs": 20794, "outputTokensPerSec": 164.91401663877164, "at": "2026-10-28T13:58:51.052Z" }, "lastUpdated": "2026-10-28T13:58:51.052Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T13:58:52.689Z" } }, "source": "engine-log" }, "process": { "sampleCount": 86, "peakRssMb": 20791, "peakCpuPercent": 274.1, "firstAtMs": 0, "lastAtMs": 470815 }, "gpu": { "available": true, "sampleCount": 86, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 86, "peakUsedMb": 31231, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] profile-cards/1 paused for help (gate_exhausted) at step \"verify\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/command-tool-availability.json preserves all 9 model-facing rosters with exact daemon line numbers, the first required-run gate note and SHA256 hashes of both source logs.", "Required gate on profile-cards/1, step verify, at 10/28/2026 13:52:45 explicitly demands run_package_script; zero captured model-facing rosters include it.", "Recorded tool calls contain no package runner; the broad registered bridge roster is not the model-visible filtered tool list." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; the underlying policy layer causing omission is not established. Preserve original model-default result and fixed score, strict planned attempt count, artifact flaws and actual judgments/performance. This supplemental eligibility supersedes earlier provisional no-exclusion notes for this trial. No source, dependency, tool permission, gate or model tuning was changed.", "reviewedAt": "2026-10-28T15:44:56.4228577Z" }, "startedAt": "2026-10-28T13:50:57.502Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T14:02:36.780Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read the required artifacts-drawer verification and all supporting artifacts, alternate workspace verification, final source/test/history files, brief and terminal result. Captured pre-edit search and all api.js mutation events were read and frozen with recording SHA256 and complete tool counts in judge-context-codemod.json. Exact frozen Enumeration completeness/Invariant discipline/Reconciled verification axes; combined selected input below 20 KiB. Judged despite failure; no new execution or source edits.", "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb", "artifactPath": "artifacts/profile-cards/tasks/1/verification.md", "contextArtifactPaths": [ "artifacts/profile-cards/tasks/1/sites.md", "artifacts/profile-cards/tasks/1/change-notes.md", "artifacts/profile-cards/tasks/1/review.md", "workspace/profile-cards/tasks/1/verification.md", "workspace/profile-cards/source/rename-brief.md", "workspace/profile-cards/package.json", "workspace/profile-cards/src/api.js", "workspace/profile-cards/src/index.js", "workspace/profile-cards/src/profile.js", "workspace/profile-cards/tests/profile.test.mjs", "workspace/profile-cards/CHANGELOG.md", "judge-context-codemod.json", "result.json" ], "artifactKind": "markdown", "scoreAxes": { "Enumeration completeness": 8, "Invariant discipline": 1, "Reconciled verification": 1 }, "justification": "A real pre-edit search and precise site list cover all seven live occurrences with justified exclusions. Later edits export getUserProfile instead of loadUser, breaking existing imports. Verification claims completion without a rescan; the required report admits no test run, while review/workspace copies falsely claim passed tests and disagree on the rename.", "artifactSha256": "a6f98095ef9f7a9906c301be72ceb047e1c4b62f8b80314ab79a4caf4eecf111", "artifactBytes": 882, "contextArtifacts": [ { "path": "artifacts/profile-cards/tasks/1/sites.md", "bytes": 763, "sha256": "b4d774ec4e33752c38a38be3e7fb3bc0c9e5b51fda0c616ac504d4b153393bf3" }, { "path": "artifacts/profile-cards/tasks/1/change-notes.md", "bytes": 519, "sha256": "4bbef997f0bbaf57478338cede891c3d9c8653ac842d99d1458a348501e2bb59" }, { "path": "artifacts/profile-cards/tasks/1/review.md", "bytes": 1761, "sha256": "0451c5efe291c5f28b06b7552f2432ef98710b1ba1fb14e1d6947e2af3141fd1" }, { "path": "workspace/profile-cards/tasks/1/verification.md", "bytes": 271, "sha256": "4acb5e4e344dc2d0f8977e99ebc25a772cea26faffe7bf3c73e03321acd5c56f" }, { "path": "workspace/profile-cards/source/rename-brief.md", "bytes": 534, "sha256": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80" }, { "path": "workspace/profile-cards/package.json", "bytes": 137, "sha256": "d69e9390722b6b5ae20c94d274faf88a16fa95d57293a7d2fcf5ecad97c0650b" }, { "path": "workspace/profile-cards/src/api.js", "bytes": 231, "sha256": "01272894ba13b3d0f0b0c2d643a48ddc1388fe666d5f17d268b6a201905a9b7a" }, { "path": "workspace/profile-cards/src/index.js", "bytes": 37, "sha256": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c" }, { "path": "workspace/profile-cards/src/profile.js", "bytes": 224, "sha256": "b7bfec2ac66313b137b25a1411df47ce17f7d1a648119343b585116d01b70819" }, { "path": "workspace/profile-cards/tests/profile.test.mjs", "bytes": 555, "sha256": "fb92b162b53605262a2032ece1962f8c5deb8ac182c75d98c4202db54402c82d" }, { "path": "workspace/profile-cards/CHANGELOG.md", "bytes": 207, "sha256": "51760c8cae86a3fd0b4ff6a0ea029d890ebc0e64045f32b7bed53e2da145e5ea" }, { "path": "judge-context-codemod.json", "bytes": 4919, "sha256": "c94e3dc480c0f5844c8c182f1085ee08b00e0ab21bff67cb2d90e506bdcd1eaf" }, { "path": "result.json", "bytes": 1457, "sha256": "74c4112bc432622b8ba3513c6f3d24434e0fdca1ea0f3ea74fae1e531e7eb727" } ], "inputBytes": 12497, "inputLimitBytes": 20480, "truncated": false, "meanScore": 3.3333333333333335 }, "systemMemory": { "sampleCount": 86, "peakUsedMb": 31231, "totalMb": 31865 }, "evidenceHashes": { "result.json": "74c4112bc432622b8ba3513c6f3d24434e0fdca1ea0f3ea74fae1e531e7eb727", "score.json": "b55c5b3ab9ed1a06c5b4dbf78a4351c3bb98833708cd51b2823c128328aa9861", "metrics.json": "e00078132d5740f3eecc9f7512a9cdcba6dbf37f8085cdcef649feb1aa85be9b", "codex-inline-judge.json": "774494203a45110c9bdc1cc48769d55538291985b1570402f1fda8b00fc36a24" }, "rubricId": "craftbook-codemod-sweep", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb", "scenarioId": "craftbook-codemod-sweep", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "craftbook-codemod-sweep stale no-write repair loop: tasks/1/verification.md had unchanged content (882 bytes) for 77s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 27/32: tasks/1/verification.md is missing required content: verification sections in order — nothing in its 882 bytes matches /^##\\s+Residual scan[\\s\\S]*^##\\s+Counts reconciled[\\s\\S]*^##\\s+Suite[\\s\\S]*^##\\s+Result/. Add that content." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34081 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 763, "score": 27, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json" } } ] }, "quality": { "score": 8.4, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 84.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34081 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 763, "score": 27, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08874574074074074 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 80 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 34081 }, { "source": "facts", "path": "outcome.durationMs", "value": 479227 } ] }, "behavior": { "score": 5, "ruleId": "B_CLEAN_HIGH_INTERVENTION", "summary": "No red flags or missing roles were observed, but the team needed at least three interventions.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 5 } ] } }, "composite": 4.6, "band": "framework-gap" } }, { "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu", "runDir": "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu", "finishedAt": "2026-10-28T14:15:45.436Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on workspace output; missing signals stayed unchanged: tests/profile.test.mjs did not pass when run with node: exit=1\nCommand failed: Files\\nodejs\\node.exe tests/profile.test.mjs\n/tests/profile.test.mjs:3\r\nimport { fetchUserData } from '../src/index.js';\r\n ^^^^^^^^^^^^^\r\nSyntaxError: The requested module '../src/index.js' does not provide an export named 'fetchUserData'\r\n at #asyncInstantiate (node:internal/modules/esm/module_job:455:21)\r\n at async ModuleJob.run (node:internal/modules/esm/module_job:553:5)\r\n at async node:internal/modules/esm/loader:647:26\r\n at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)\r\n\r\nNode.js v24.21.0, tests/verify-sweep.mjs did not pass when run with node: exit=1\nCommand failed: Files\\nodejs\\node.exe tests/verify-sweep.mjs\nnode:internal/modules/run_main:107\r\n triggerUncaughtException(\r\n ^\r\n\r\nAssertionError [ERR_ASSERTION]: the old name survives outside the exclusions in: tests/profile.test.mjs\r\n+ actual - expected\r\n\r\n+ [\r\n+ 'tests/profile.test.mjs'\r\n+ ]\r\n- []\r\n\r\n at /tests/verify-sweep.mjs:32:8\r\n at ModuleJob.run (node:internal/modules/esm/module_job:561:25)\r\n at async node:internal/modules/esm/loader:647:26\r\n at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) {\r\n generatedMessage: false,\r\n code: 'ERR_ASSERTION',\r\n actual: [ 'tests/profile.test.mjs' ],\r\n expected: [],\r\n operator: 'deepStrictEqual',\r\n diff: 'simple'\r\n}\r\n\r\nNode.js v24.21.0, task sourced from craftbook codemod-sweep has not reached a terminal step, tests/profile.test.mjs contains forbidden content: the old name is gone from the test call sites (matched \"fetchUserData\"). Last failure: tests/profile.test.mjs did not pass when run with node: exit=1 Command failed: Files\\nodejs\\node.exe tests/profile.test.mjs /tests/profile.test.mjs:3 import { fetchUserData } from '../src/index.js'; ^^^^^^^^^^^^^ SyntaxError: The requested module '../src/index.js' does not provide an export named 'fetchUserData' at #asyncInstantiate (node:internal/modules/esm/module_job:455:21) at async ModuleJob.run (node:internal/modules/esm/module_job:553:5) at async node:internal/modules/esm/loader:647:26 at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) Node.js v24.21.0", "durationMs": 1007790, "rawFixedComposite": 5.3, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 3.6666666666666665, "judgeArtifactCandidates": [ "artifacts/profile-cards/tasks/1/verification.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 119.22, "nativePrefillTokensPerSec": 1236.6, "inputTokens": 233296, "outputTokens": 27722, "peakRssMb": 20509, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 1007790, "derived": { "durationMs": 995810, "meanTokensPerSec": 119.22, "genTokensPerSec": 119.22, "genTokens": 27722, "promptTokensPerSec": 1236.6, "peakRssMb": 20509, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 146330, "output": 4429 } } }, "usage": { "available": true, "totalInputTokens": 233296, "totalOutputTokens": 27722, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 146330, "todayTokensCached": 142803, "todayTokensOut": 4429, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 146330, "totalTokensCached": 142803, "totalTokensOut": 4429, "totalCost": 0, "medianOutputTokensPerSec": 137.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 137.9, "turns": 11 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16234, "cachedInputTokens": 15798, "outputTokens": 263, "cost": 0, "durationMs": 2678, "outputTokensPerSec": 144.79121879942656, "at": "2026-10-28T14:02:17.660Z" }, "lastUpdated": "2026-10-28T14:02:17.660Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:15:36.833Z" } }, "source": "engine-log" }, "process": { "sampleCount": 179, "peakRssMb": 20509, "peakCpuPercent": 262.4, "firstAtMs": 1, "lastAtMs": 994801 }, "gpu": { "available": true, "sampleCount": 179, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 179, "peakUsedMb": 31297, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] profile-cards/1 paused for help (gate_exhausted) at step \"verify\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/command-tool-availability.json preserves all 8 model-facing rosters with exact daemon line numbers, the first required-run gate note and SHA256 hashes of both source logs.", "Required gate on profile-cards/1, step verify, at 10/28/2026 14:00:41 explicitly demands run_package_script; zero captured model-facing rosters include it.", "Recorded tool calls contain no package runner; the broad registered bridge roster is not the model-visible filtered tool list." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; the underlying policy layer causing omission is not established. Preserve original model-default result and fixed score, strict planned attempt count, artifact flaws and actual judgments/performance. This supplemental eligibility supersedes earlier provisional no-exclusion notes for this trial. No source, dependency, tool permission, gate or model tuning was changed.", "reviewedAt": "2026-10-28T15:44:56.4338339Z" }, "startedAt": "2026-10-28T13:58:57.646Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T14:20:34.101Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read the complete third verification, site/change/review artifacts, final source/test/history files and brief. Captured actual initial grep, first source mutation, all test-file mutation events, complete tool counts and terminal failure are frozen with recording SHA256 in judge-context-codemod.json. Exact frozen Enumeration completeness/Invariant discipline/Reconciled verification axes; selected input below 20 KiB. No new execution or source edits.", "suiteId": "developer", "scenarioId": "craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu", "artifactPath": "artifacts/profile-cards/tasks/1/verification.md", "contextArtifactPaths": [ "artifacts/profile-cards/tasks/1/sites.md", "artifacts/profile-cards/tasks/1/change-notes.md", "artifacts/profile-cards/tasks/1/review.md", "workspace/profile-cards/source/rename-brief.md", "workspace/profile-cards/package.json", "workspace/profile-cards/src/api.js", "workspace/profile-cards/src/index.js", "workspace/profile-cards/src/profile.js", "workspace/profile-cards/tests/profile.test.mjs", "workspace/profile-cards/CHANGELOG.md", "judge-context-codemod.json" ], "artifactKind": "markdown", "scoreAxes": { "Enumeration completeness": 8, "Invariant discipline": 2, "Reconciled verification": 1 }, "justification": "Real pre-edit search enumerates all 11 occurrences and explains the two excluded files. The model then reverts both test sites to the old import/call, breaking the suite. Claimed 9 changed + 2 excluded mixes occurrences with files; there are four excluded occurrences. Zero residual and PASS claims contradict saved tests and the captured failure.", "artifactSha256": "3fa0b32232e58ce59f997c5c61582714b67f37d2e78ae2f2da09edc95f7d6a4f", "artifactBytes": 708, "contextArtifacts": [ { "path": "artifacts/profile-cards/tasks/1/sites.md", "bytes": 1152, "sha256": "9f020ecf4d56800c90483568a6bfd2a00dae49a630276006589f4e4899a400cf" }, { "path": "artifacts/profile-cards/tasks/1/change-notes.md", "bytes": 519, "sha256": "21552571c17c862f1b4f69c5f7c1e9ddabf37a2ebfd0aaaf0ee52f38831ec3c3" }, { "path": "artifacts/profile-cards/tasks/1/review.md", "bytes": 814, "sha256": "a82ad2ac078c55febf6c794f50b473a01881266119dd09b5ef83577ea3faa6bb" }, { "path": "workspace/profile-cards/source/rename-brief.md", "bytes": 534, "sha256": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80" }, { "path": "workspace/profile-cards/package.json", "bytes": 137, "sha256": "d69e9390722b6b5ae20c94d274faf88a16fa95d57293a7d2fcf5ecad97c0650b" }, { "path": "workspace/profile-cards/src/api.js", "bytes": 225, "sha256": "bd78e33b9cf74aec788b859016f4a43e161b0173bd06add93ecc140b791c7a0d" }, { "path": "workspace/profile-cards/src/index.js", "bytes": 37, "sha256": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c" }, { "path": "workspace/profile-cards/src/profile.js", "bytes": 225, "sha256": "a723dc6e94beda9791e150daf98b27ae01f94968a7ffecabd4e0ad736641f3b0" }, { "path": "workspace/profile-cards/tests/profile.test.mjs", "bytes": 566, "sha256": "e607dd92c736e8fca735835ae316d8936baecca5ba9a93ab2ec2ff00f42893bc" }, { "path": "workspace/profile-cards/CHANGELOG.md", "bytes": 207, "sha256": "51760c8cae86a3fd0b4ff6a0ea029d890ebc0e64045f32b7bed53e2da145e5ea" }, { "path": "judge-context-codemod.json", "bytes": 5879, "sha256": "bebe919112f8553ca6dcdb0210f9a08eb39ea1e9c82c069a5ae9b353f1dd2dda" } ], "inputBytes": 11003, "inputLimitBytes": 20480, "truncated": false, "meanScore": 3.6666666666666665 }, "systemMemory": { "sampleCount": 179, "peakUsedMb": 31297, "totalMb": 31865 }, "evidenceHashes": { "result.json": "fa18bb972431b555e8f391b585257e4e6c352cbf387e87bf3d29b171b50d5216", "score.json": "b8b9f7263e7962a189490a4f3544e814388e681fa7947cb23629b962a7d0cfa2", "metrics.json": "731a206a99e7fe749015eb536662e9226f3b9db6bf9f8321f74b67c32f01bc5c", "codex-inline-judge.json": "5549a5df78227fa406f9d18f33de11efe45cf35e1c3b269bcbe3dd1984e5d93a" }, "rubricId": "craftbook-codemod-sweep", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu", "scenarioId": "craftbook-codemod-sweep", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on workspace output; missing signals stayed unchanged: tests/profile.test.mjs did not pass when run with node: exit=1\nCommand failed: Files\\nodejs\\node.exe tests/profile.test.mjs\n/tests/profile.test.mjs:3\r\nimport { fetchUserData } from '../src/index.js';\r\n ^^^^^^^^^^^^^\r\nSyntaxError: The requested module '../src/index.js' does not provide an export named 'fetchUserData'\r\n at #asyncInstantiate (node:internal/modules/esm/module_job:455:21)\r\n at async ModuleJob.run (node:internal/modules/esm/module_job:553:5)\r\n at async node:internal/modules/esm/loader:647:26\r\n at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)\r\n\r\nNode.js v24.21.0, tests/verify-sweep.mjs did not pass when run with node: exit=1\nCommand failed: Files\\nodejs\\node.exe tests/verify-sweep.mjs\nnode:internal/modules/run_main:107\r\n triggerUncaughtException(\r\n ^\r\n\r\nAssertionError [ERR_ASSERTION]: the old name survives outside the exclusions in: tests/profile.test.mjs\r\n+ actual - expected\r\n\r\n+ [\r\n+ 'tests/profile.test.mjs'\r\n+ ]\r\n- []\r\n\r\n at /tests/verify-sweep.mjs:32:8\r\n at ModuleJob.run (node:internal/modules/esm/module_job:561:25)\r\n at async node:internal/modules/esm/loader:647:26\r\n at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) {\r\n generatedMessage: false,\r\n code: 'ERR_ASSERTION',\r\n actual: [ 'tests/profile.test.mjs' ],\r\n expected: [],\r\n operator: 'deepStrictEqual',\r\n diff: 'simple'\r\n}\r\n\r\nNode.js v24.21.0, task sourced from craftbook codemod-sweep has not reached a terminal step, tests/profile.test.mjs contains forbidden content: the old name is gone from the test call sites (matched \"fetchUserData\"). Last failure: tests/profile.test.mjs did not pass when run with node: exit=1 Command failed: Files\\nodejs\\node.exe tests/profile.test.mjs /tests/profile.test.mjs:3 import { fetchUserData } from '../src/index.js'; ^^^^^^^^^^^^^ SyntaxError: The requested module '../src/index.js' does not provide an export named 'fetchUserData' at #asyncInstantiate (node:internal/modules/esm/module_job:455:21) at async ModuleJob.run (node:internal/modules/esm/module_job:553:5) at async node:internal/modules/esm/loader:647:26 at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) Node.js v24.21.0" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 31891 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 1152, "score": 27, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not reached a terminal step" } } ] }, "quality": { "score": 8.4, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 84.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 31891 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 1152, "score": 27, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not reached a terminal step" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.18662777777777778 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 199 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 31891 }, { "source": "facts", "path": "outcome.durationMs", "value": 1007790 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.3, "band": "needs-tuning" } }, { "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct", "runDir": "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct", "finishedAt": "2026-10-28T14:27:36.824Z", "rawSuccess": false, "rawReason": "craftbook-refactor-module stale no-write repair loop: tasks/1/plan.md had unchanged content (1960 bytes) for 127s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 19/30: tasks/1/plan.md cites 1 source(s) that do not exist: `./normalize-name.js` — every cited path must resolve to a real file in the workspace (no fabricated citations).", "durationMs": 707259, "rawFixedComposite": 4.8, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 1.3333333333333333, "judgeArtifactCandidates": [ "artifacts/crewlog/tasks/1/change-notes.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 128.83, "nativePrefillTokensPerSec": 1734.21, "inputTokens": 544801, "outputTokens": 30389, "peakRssMb": 21846, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 707259, "derived": { "durationMs": 699347, "meanTokensPerSec": 128.83, "genTokensPerSec": 128.83, "genTokens": 30389, "promptTokensPerSec": 1734.21, "peakRssMb": 21846, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 117882, "output": 4271 } } }, "usage": { "available": true, "totalInputTokens": 544801, "totalOutputTokens": 30389, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 117882, "todayTokensCached": 106431, "todayTokensOut": 4271, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 117882, "totalTokensCached": 106431, "totalTokensOut": 4271, "totalCost": 0, "medianOutputTokensPerSec": 115.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 115.9, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 27318, "cachedInputTokens": 26703, "outputTokens": 200, "cost": 0, "durationMs": 22165, "outputTokensPerSec": 111.15952229334941, "at": "2026-10-28T14:25:19.208Z" }, "lastUpdated": "2026-10-28T14:25:19.208Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:27:32.501Z" } }, "source": "engine-log" }, "process": { "sampleCount": 127, "peakRssMb": 21846, "peakCpuPercent": 262.8, "firstAtMs": 1, "lastAtMs": 698336 }, "gpu": { "available": true, "sampleCount": 127, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 127, "peakUsedMb": 31130, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] crewlog/1 paused for help (step_stalled) at step \"baseline\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/command-tool-availability.json preserves all 7 model-facing rosters with exact daemon line numbers, the first required-run gate note and SHA256 hashes of both source logs.", "Required gate on crewlog/1, step baseline, at 10/28/2026 14:17:06 explicitly demands run_package_script; zero captured model-facing rosters include it.", "Recorded tool calls contain no package runner; the broad registered bridge roster is not the model-visible filtered tool list." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; the underlying policy layer causing omission is not established. Preserve original model-default result and fixed score, strict planned attempt count, artifact flaws and actual judgments/performance. This supplemental eligibility supersedes earlier provisional no-exclusion notes for this trial. No source, dependency, tool permission, gate or model tuning was changed.", "reviewedAt": "2026-10-28T15:44:56.4442147Z" }, "startedAt": "2026-10-28T14:15:49.565Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T14:34:36.569Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read all five final deliverables, brief, package, all three source files and tests. Selected actual tool events, complete captured tool counts, terminal result and latest fixed facts sniff are frozen with recording SHA256 in judge-context-refactor.json. Applied the exact frozen Guardrail-first discipline/Reversible-stage execution/Genuine deduplication axes. No new test execution. Guard-rejected source writes are distinguished from successful writes; all selected input is below 20 KiB without clipping.", "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct", "artifactPath": "artifacts/crewlog/tasks/1/change-notes.md", "contextArtifactPaths": [ "artifacts/crewlog/tasks/1/baseline.md", "artifacts/crewlog/tasks/1/plan.md", "artifacts/crewlog/tasks/1/verification.md", "artifacts/crewlog/tasks/1/review.md", "workspace/crewlog/source/refactor-brief.md", "workspace/crewlog/package.json", "workspace/crewlog/src/format-name.js", "workspace/crewlog/src/report.js", "workspace/crewlog/src/normalize-name.js", "workspace/crewlog/tests/format.test.mjs", "workspace/crewlog/tests/report.test.mjs", "workspace/crewlog/tests/characterization.test.mjs", "judge-context-refactor.json" ], "artifactKind": "markdown", "scoreAxes": { "Guardrail-first discipline": 2, "Reversible-stage execution": 1, "Genuine deduplication": 1 }, "justification": "Baseline considers characterization and a count method, but captured tools show no suite run and new tests use unimported describe. Claims of green stages and completion are unsupported. The shared helper is unused: both original copies remain, so duplication rises to three. Write guards rejected the attempted shorter wrapper; the reports still claim success.", "artifactSha256": "c6474cac677f66f99208ac3078e36c5d7cb4c37baab7e4f1462dc44aa6be705b", "artifactBytes": 1207, "contextArtifacts": [ { "path": "artifacts/crewlog/tasks/1/baseline.md", "bytes": 1691, "sha256": "975ffd8358e077d6a16938fb3ea41dc9c33c66b33c6228f6d7ee3e42cb340493" }, { "path": "artifacts/crewlog/tasks/1/plan.md", "bytes": 1960, "sha256": "384240904edc604761507fe7c981fb03870574f715b6688739d177287d03c884" }, { "path": "artifacts/crewlog/tasks/1/verification.md", "bytes": 1057, "sha256": "b6a1b442b171a8f6267175cd6dfe15e6f4c1158bf25c04f0a8df9a70bed36a99" }, { "path": "artifacts/crewlog/tasks/1/review.md", "bytes": 820, "sha256": "45d7e843abfee1514e8bc82951a3430ce091050d7568db639cfb043c9795908e" }, { "path": "workspace/crewlog/source/refactor-brief.md", "bytes": 720, "sha256": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34" }, { "path": "workspace/crewlog/package.json", "bytes": 131, "sha256": "06c2f0dc9cbe2ed0b953f8e1e9f3fa7a41326d0b7222b4d26aa35fef2de25c1b" }, { "path": "workspace/crewlog/src/format-name.js", "bytes": 241, "sha256": "4779137dd7acff80d97a674b85055f0344eba270160f090dc5798c9867be7b63" }, { "path": "workspace/crewlog/src/report.js", "bytes": 446, "sha256": "0ea85cd02142f8d5474afea2f71ea3f39d72532e2676b84f2aed299b7bbbb791" }, { "path": "workspace/crewlog/src/normalize-name.js", "bytes": 243, "sha256": "d1dddc05eaf390790717346986ee6417f509d0fce84f01c36c95e26665bf8595" }, { "path": "workspace/crewlog/tests/format.test.mjs", "bytes": 234, "sha256": "80c14472bc71dee5f53f76e4d1428c053ee96bdd68afa984bd583578b4b981ce" }, { "path": "workspace/crewlog/tests/report.test.mjs", "bytes": 693, "sha256": "3e3a3ac8ac564c254e4127b74b745be4ada01d81c263b4c514f47c2225b04612" }, { "path": "workspace/crewlog/tests/characterization.test.mjs", "bytes": 979, "sha256": "14373970f0d38c9c9f0b397ae0066f2ccc124d8a95366814640332296e454def" }, { "path": "judge-context-refactor.json", "bytes": 5434, "sha256": "5638aaf42cfd77ddf9a5de819a1a4b49ddd2b2cb0d59bb3a4206437eca94595c" } ], "inputBytes": 15856, "inputLimitBytes": 20480, "truncated": false, "meanScore": 1.3333333333333333 }, "systemMemory": { "sampleCount": 127, "peakUsedMb": 31130, "totalMb": 31865 }, "evidenceHashes": { "result.json": "896ca1f0a0ce811b94119d7a3a469dec31d63d43188a4c8046557bc796afaa34", "score.json": "33b596d98c9f1535aaafb9a867d18ffa00f30043f6fa552845d41ee27a7d86ae", "metrics.json": "ec2778c23a451034d025b354a26c9f3415a84e9c8be5b5ca880837861d2ca92a", "codex-inline-judge.json": "94bb7f8917d23cbdb97851762f0880b24857a37a179f64cb334efbb5d116b562" }, "rubricId": "craftbook-refactor-module", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct", "scenarioId": "craftbook-refactor-module", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "craftbook-refactor-module stale no-write repair loop: tasks/1/plan.md had unchanged content (1960 bytes) for 127s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 19/30: tasks/1/plan.md cites 1 source(s) that do not exist: `./normalize-name.js` — every cited path must resolve to a real file in the workspace (no fabricated citations)." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23703 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 1691, "score": 19, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not reached a terminal step" } } ] }, "quality": { "score": 6.3, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 63.3% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23703 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 1691, "score": 19, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not reached a terminal step" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.09823041666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 125 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23703 }, { "source": "facts", "path": "outcome.durationMs", "value": 707259 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.8, "band": "framework-gap" } }, { "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg", "runDir": "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg", "finishedAt": "2026-10-28T14:41:01.587Z", "rawSuccess": false, "rawReason": "craftbook-refactor-module stale no-write repair loop: tasks/1/plan.md had unchanged content (745 bytes) for 150s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 18/30: tasks/1/plan.md cites 1 source(s) that do not exist: `src/normalize-name.js` — every cited path must resolve to a real file in the workspace (no fabricated citations).", "durationMs": 804181, "rawFixedComposite": 4.7, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 0.6666666666666666, "judgeArtifactCandidates": [ "artifacts/crewlog/tasks/1/change-notes.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 129.41, "nativePrefillTokensPerSec": 1719.12, "inputTokens": 579179, "outputTokens": 41400, "peakRssMb": 21621, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10521, "gpuMemoryTotalMb": 12227, "durationMs": 804181, "derived": { "durationMs": 795335, "meanTokensPerSec": 129.41, "genTokensPerSec": 129.41, "genTokens": 41400, "promptTokensPerSec": 1719.12, "peakRssMb": 21621, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 146698, "output": 7661 } } }, "usage": { "available": true, "totalInputTokens": 579179, "totalOutputTokens": 41400, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 146698, "todayTokensCached": 131807, "todayTokensOut": 7661, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 146698, "totalTokensCached": 131807, "totalTokensOut": 7661, "totalCost": 0, "medianOutputTokensPerSec": 128, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 128, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 23523, "cachedInputTokens": 23113, "outputTokens": 438, "cost": 0, "durationMs": 31129, "outputTokensPerSec": 131.0246608200285, "at": "2026-10-28T14:35:49.028Z" }, "lastUpdated": "2026-10-28T14:35:49.028Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T14:40:55.969Z" } }, "source": "engine-log" }, "process": { "sampleCount": 144, "peakRssMb": 21621, "peakCpuPercent": 266, "firstAtMs": 0, "lastAtMs": 794312 }, "gpu": { "available": true, "sampleCount": 144, "peakUtilPercent": 100, "peakMemUsedMb": 10521, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 144, "peakUsedMb": 31504, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] crewlog/1 paused for help (step_stalled) at step \"baseline\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/command-tool-availability.json preserves all 11 model-facing rosters with exact daemon line numbers, the first required-run gate note and SHA256 hashes of both source logs.", "Required gate on crewlog/1, step baseline, at 10/28/2026 14:28:37 explicitly demands run_package_script; zero captured model-facing rosters include it.", "Recorded tool calls contain no package runner; the broad registered bridge roster is not the model-visible filtered tool list." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; the underlying policy layer causing omission is not established. Preserve original model-default result and fixed score, strict planned attempt count, artifact flaws and actual judgments/performance. This supplemental eligibility supersedes earlier provisional no-exclusion notes for this trial. No source, dependency, tool permission, gate or model tuning was changed.", "reviewedAt": "2026-10-28T15:44:56.4535670Z" }, "startedAt": "2026-10-28T14:27:37.406Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T14:50:58.391Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read all five final deliverables and the trial's own brief, package, source and tests. Reviewed actual characterization-write and subsequent source-read chronology, complete captured tool counts, final source inventory and terminal result frozen with recording SHA256 in judge-context-refactor.json. Applied the three exact frozen axes independently. No new execution, no inherited source-write-guard diagnosis, and no clipping of selected inputs under 20 KiB.", "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg", "artifactPath": "artifacts/crewlog/tasks/1/change-notes.md", "contextArtifactPaths": [ "artifacts/crewlog/tasks/1/baseline.md", "artifacts/crewlog/tasks/1/plan.md", "artifacts/crewlog/tasks/1/verification.md", "artifacts/crewlog/tasks/1/review.md", "workspace/crewlog/source/refactor-brief.md", "workspace/crewlog/package.json", "workspace/crewlog/src/format-name.js", "workspace/crewlog/src/report.js", "workspace/crewlog/tests/format.test.mjs", "workspace/crewlog/tests/report.test.mjs", "workspace/crewlog/tests/characterization.test.mjs", "judge-context-refactor.json" ], "artifactKind": "markdown", "scoreAxes": { "Guardrail-first discipline": 1, "Reversible-stage execution": 1, "Genuine deduplication": 0 }, "justification": "Characterization was written before reading source, with nonexistent relative imports/export and wrong lowercase/character-stripping expectations. No captured suite run supports GREEN claims. Change notes admit no changes, but review falsely claims verified extraction. The shared module is absent and both original copies remain; no stages were executed.", "artifactSha256": "da08288a7f61c58b2e4e01fdb2f148f664bb8b0cd9a3ef75a8bcc44c40813938", "artifactBytes": 1356, "contextArtifacts": [ { "path": "artifacts/crewlog/tasks/1/baseline.md", "bytes": 1452, "sha256": "378449015566ebdfa544af0e92c903cee40e69ca62c14faf0fb8658d34e934ce" }, { "path": "artifacts/crewlog/tasks/1/plan.md", "bytes": 745, "sha256": "f68906cdf911fd14f9178df6394b66c2bd1323f29d40c1530e19e5216c1727e6" }, { "path": "artifacts/crewlog/tasks/1/verification.md", "bytes": 588, "sha256": "40fdc9f35219856df131027bbe7eb88e6bea26d234667d385e6bd354b0e28dac" }, { "path": "artifacts/crewlog/tasks/1/review.md", "bytes": 278, "sha256": "cb88aecf12f3ee6ac1516cf4ec596a265a0efb5169773d59d8b9c443fc48e90b" }, { "path": "workspace/crewlog/source/refactor-brief.md", "bytes": 720, "sha256": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34" }, { "path": "workspace/crewlog/package.json", "bytes": 131, "sha256": "06c2f0dc9cbe2ed0b953f8e1e9f3fa7a41326d0b7222b4d26aa35fef2de25c1b" }, { "path": "workspace/crewlog/src/format-name.js", "bytes": 241, "sha256": "4779137dd7acff80d97a674b85055f0344eba270160f090dc5798c9867be7b63" }, { "path": "workspace/crewlog/src/report.js", "bytes": 446, "sha256": "0ea85cd02142f8d5474afea2f71ea3f39d72532e2676b84f2aed299b7bbbb791" }, { "path": "workspace/crewlog/tests/format.test.mjs", "bytes": 234, "sha256": "80c14472bc71dee5f53f76e4d1428c053ee96bdd68afa984bd583578b4b981ce" }, { "path": "workspace/crewlog/tests/report.test.mjs", "bytes": 259, "sha256": "fc6456020a060a454549183e3ea594d18e313bc33e18375969b5fe2a70853351" }, { "path": "workspace/crewlog/tests/characterization.test.mjs", "bytes": 1623, "sha256": "32418eed50fe2df7731620914b2a6b5455fbd6b41afc88015bd15d35482ec20b" }, { "path": "judge-context-refactor.json", "bytes": 6773, "sha256": "2acc61ebc2c02cca2effbf49932f7f30fc96ee3f5657a1712e5a7e8f8191aeb8" } ], "inputBytes": 14846, "inputLimitBytes": 20480, "truncated": false, "meanScore": 0.6666666666666666 }, "systemMemory": { "sampleCount": 144, "peakUsedMb": 31504, "totalMb": 31865 }, "evidenceHashes": { "result.json": "50e4e011f3e28c26643eac29601c8d0258bff3489034d1cacfb37d2969d2beea", "score.json": "6760dc43c8caa540f4b3d43fe003c9ca7d75fca3aa7d8fd2df0a0815f2a56853", "metrics.json": "6112d207f8ba6da0aa76252ac9d667ae4f4ab4a760b533d2cb9659d1db2a3bb4", "codex-inline-judge.json": "c76da1b1d42ba3da51c43cc6dd15685499f3d551e6b71af92930090e651ec678" }, "rubricId": "craftbook-refactor-module", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg", "scenarioId": "craftbook-refactor-module", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "craftbook-refactor-module stale no-write repair loop: tasks/1/plan.md had unchanged content (745 bytes) for 150s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 18/30: tasks/1/plan.md cites 1 source(s) that do not exist: `src/normalize-name.js` — every cited path must resolve to a real file in the workspace (no fabricated citations)." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23774 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 1452, "score": 18, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not reached a terminal step" } } ] }, "quality": { "score": 6, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 60% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23774 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 1452, "score": 18, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not reached a terminal step" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.11169180555555555 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 141 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 23774 }, { "source": "facts", "path": "outcome.durationMs", "value": 804181 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.7, "band": "framework-gap" } }, { "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe", "runDir": "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe", "finishedAt": "2026-10-28T15:07:33.451Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on tasks/1/plan.md; missing signals stayed unchanged: tasks/1/plan.md is missing required content: plan sections in order — nothing in its 1829 bytes matches /^##\\s+Target[\\s\\S]*^##\\s+Stages[\\s\\S]*^##\\s+Acceptance criteria/. Add that content.. Last failure: tasks/1/plan.md is missing required content: plan sections in order — nothing in its 1829 bytes matches /^##\\s+Target[\\s\\S]*^##\\s+Stages[\\s\\S]*^##\\s+Acceptance criteria/. Add that content.", "durationMs": 1590626, "rawFixedComposite": 5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 3.6666666666666665, "judgeArtifactCandidates": [ "artifacts/crewlog/tasks/1/change-notes.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 120.37, "nativePrefillTokensPerSec": 1608.86, "inputTokens": 996921, "outputTokens": 28983, "peakRssMb": 21368, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 1590626, "derived": { "durationMs": 1577642, "meanTokensPerSec": 120.37, "genTokensPerSec": 120.37, "genTokens": 28983, "promptTokensPerSec": 1608.86, "peakRssMb": 21368, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 224080, "output": 8675 } } }, "usage": { "available": true, "totalInputTokens": 996921, "totalOutputTokens": 28983, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 224080, "todayTokensCached": 139973, "todayTokensOut": 8675, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 224080, "totalTokensCached": 139973, "totalTokensOut": 8675, "totalCost": 0, "medianOutputTokensPerSec": 121.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 121.2, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 28755, "cachedInputTokens": 28706, "outputTokens": 516, "cost": 0, "durationMs": 46966, "outputTokensPerSec": 147.4349046267936, "at": "2026-10-28T14:50:38.003Z" }, "lastUpdated": "2026-10-28T14:50:38.003Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:07:23.884Z" } }, "source": "engine-log" }, "process": { "sampleCount": 285, "peakRssMb": 21368, "peakCpuPercent": 262.7, "firstAtMs": 1, "lastAtMs": 1576353 }, "gpu": { "available": true, "sampleCount": 285, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 285, "peakUsedMb": 31239, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] crewlog/1 paused for help (step_stalled) at step \"baseline\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/command-tool-availability.json preserves all 9 model-facing rosters with exact daemon line numbers, the first required-run gate note and SHA256 hashes of both source logs.", "Required gate on crewlog/1, step baseline, at 10/28/2026 14:42:16 explicitly demands run_package_script; zero captured model-facing rosters include it.", "Recorded tool calls contain no package runner; the broad registered bridge roster is not the model-visible filtered tool list." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; the underlying policy layer causing omission is not established. Preserve original model-default result and fixed score, strict planned attempt count, artifact flaws and actual judgments/performance. This supplemental eligibility supersedes earlier provisional no-exclusion notes for this trial. No source, dependency, tool permission, gate or model tuning was changed.", "reviewedAt": "2026-10-28T15:44:56.4629296Z" }, "startedAt": "2026-10-28T14:41:02.825Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T15:12:47.216Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read all five final deliverables, own brief/package, all source files and all five tests. Captured helper creation, rejected shorter wrapper, successful padded replacement and report call-site update are frozen with complete tool counts and recording SHA256 in judge-context-refactor.json. The saved terminal oracle independently confirms behavior and deduplication but fails the suite. Applied the exact frozen axes, crediting working extraction separately from verification failures. No new execution or clipping; combined selected input below 20 KiB.", "suiteId": "developer", "scenarioId": "craftbook-refactor-module", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe", "artifactPath": "artifacts/crewlog/tasks/1/change-notes.md", "contextArtifactPaths": [ "artifacts/crewlog/tasks/1/baseline.md", "artifacts/crewlog/tasks/1/plan.md", "artifacts/crewlog/tasks/1/verification.md", "artifacts/crewlog/tasks/1/review.md", "workspace/crewlog/source/refactor-brief.md", "workspace/crewlog/package.json", "workspace/crewlog/src/format-name.js", "workspace/crewlog/src/report.js", "workspace/crewlog/src/normalize-name.js", "workspace/crewlog/tests/format.test.mjs", "workspace/crewlog/tests/report.test.mjs", "workspace/crewlog/tests/characterization.test.mjs", "workspace/crewlog/tests/format-characterization.test.mjs", "workspace/crewlog/tests/report-characterization.test.mjs", "judge-context-refactor.json" ], "artifactKind": "markdown", "scoreAxes": { "Guardrail-first discipline": 2, "Reversible-stage execution": 3, "Genuine deduplication": 6 }, "justification": "Both call sites use one correct shared helper; captured oracle confirms behavior and deduplication. Small edits occurred, but no captured suite execution supports green-stage claims. An earlier invalid characterization file remains and the harness suite fails. Added valid edge tests help; the review is blank and guard-induced padding is an unrecorded deviation.", "artifactSha256": "6dab4f1df57ce6e55a194542f33716ca6e4312451c8d7961e0b71050b968e40a", "artifactBytes": 656, "contextArtifacts": [ { "path": "artifacts/crewlog/tasks/1/baseline.md", "bytes": 1722, "sha256": "bf5f5dc9f00e9db1bc62c3c5d323e31987f74145a2d56786b4c02628ddb4331c" }, { "path": "artifacts/crewlog/tasks/1/plan.md", "bytes": 1829, "sha256": "a54eae84114c26d3c278edb8bf5b23e2e47f4216314e09dae84cec64eaca04dd" }, { "path": "artifacts/crewlog/tasks/1/verification.md", "bytes": 1007, "sha256": "458ddd1dd6109e4a828887c094940d06ca1c86adf4c230be72527656cb75e016" }, { "path": "artifacts/crewlog/tasks/1/review.md", "bytes": 514, "sha256": "ca7cf9f39d0e536914d9e3ac63941079173c2c6efdc9ddc0fcc90269db70b2fb" }, { "path": "workspace/crewlog/source/refactor-brief.md", "bytes": 720, "sha256": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34" }, { "path": "workspace/crewlog/package.json", "bytes": 131, "sha256": "06c2f0dc9cbe2ed0b953f8e1e9f3fa7a41326d0b7222b4d26aa35fef2de25c1b" }, { "path": "workspace/crewlog/src/format-name.js", "bytes": 796, "sha256": "7c0c3dbc8e96c5cbe1db84171e5790d2a298a696b00e7502c51c5c992a19f3ab" }, { "path": "workspace/crewlog/src/report.js", "bytes": 477, "sha256": "142ddde91160542a93cc2a3f1093cb70c30dcff122ef3fee4cc5c32e755ce3e3" }, { "path": "workspace/crewlog/src/normalize-name.js", "bytes": 243, "sha256": "d1dddc05eaf390790717346986ee6417f509d0fce84f01c36c95e26665bf8595" }, { "path": "workspace/crewlog/tests/format.test.mjs", "bytes": 234, "sha256": "80c14472bc71dee5f53f76e4d1428c053ee96bdd68afa984bd583578b4b981ce" }, { "path": "workspace/crewlog/tests/report.test.mjs", "bytes": 259, "sha256": "fc6456020a060a454549183e3ea594d18e313bc33e18375969b5fe2a70853351" }, { "path": "workspace/crewlog/tests/characterization.test.mjs", "bytes": 1010, "sha256": "40d664c0f5ecd1380c60f623a4d466bd08d61085c6896bc5f9ab930f76ec5490" }, { "path": "workspace/crewlog/tests/format-characterization.test.mjs", "bytes": 616, "sha256": "e9646d40e6780303b3effc42a854bb825d88a88d2b07db7cc8225d85db8d5866" }, { "path": "workspace/crewlog/tests/report-characterization.test.mjs", "bytes": 737, "sha256": "6764ba19c9ec9dfd51243dccb292c4974f39068f0692332e7052090337c82870" }, { "path": "judge-context-refactor.json", "bytes": 5805, "sha256": "d7f822ef086306307b34634b739bb19f51b86f32eff3fadee0d988ea91d0d39f" } ], "inputBytes": 16756, "inputLimitBytes": 20480, "truncated": false, "meanScore": 3.6666666666666665 }, "systemMemory": { "sampleCount": 285, "peakUsedMb": 31239, "totalMb": 31865 }, "evidenceHashes": { "result.json": "fcdb3c5b14510a714c6b817bef9c9b14b80b3ffbfae6076a04975971c1c20260", "score.json": "19b8219df8c13a28ffb9ad7b8c5c848ec3730534419b0cce22c52e763086ca99", "metrics.json": "85b181781a72d57477550ad748b92b111739f4c8629de4bd36502b0590d8b992", "codex-inline-judge.json": "c6c48ded50dfd26c6fc20b62a6b6fea90fe3102b4f259f1c576dbf7784aa0691" }, "rubricId": "craftbook-refactor-module", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe", "scenarioId": "craftbook-refactor-module", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on tasks/1/plan.md; missing signals stayed unchanged: tasks/1/plan.md is missing required content: plan sections in order — nothing in its 1829 bytes matches /^##\\s+Target[\\s\\S]*^##\\s+Stages[\\s\\S]*^##\\s+Acceptance criteria/. Add that content.. Last failure: tasks/1/plan.md is missing required content: plan sections in order — nothing in its 1829 bytes matches /^##\\s+Target[\\s\\S]*^##\\s+Stages[\\s\\S]*^##\\s+Acceptance criteria/. Add that content." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 22683 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 1722, "score": 22, "scoreMax": 30, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json" } } ] }, "quality": { "score": 7.3, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 73.3% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 22683 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 1722, "score": 22, "scoreMax": 30, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.22092027777777778 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 179 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 22683 }, { "source": "facts", "path": "outcome.durationMs", "value": 1590626 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5, "band": "needs-tuning" } }, { "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs", "runDir": "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs", "finishedAt": "2026-10-28T15:36:24.771Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on workspace output; missing signals stayed unchanged: task sourced from craftbook bug-fix-tdd has not reached a terminal step. Last failure: task sourced from craftbook bug-fix-tdd has not reached a terminal step", "durationMs": 1727753, "rawFixedComposite": 5.6, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 4.333333333333333, "judgeArtifactCandidates": [ "artifacts/orderly-pricing/tasks/1/fix-notes.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 121.69, "nativePrefillTokensPerSec": 1665.83, "inputTokens": 1419591, "outputTokens": 40876, "peakRssMb": 21213, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10519, "gpuMemoryTotalMb": 12227, "durationMs": 1727753, "derived": { "durationMs": 1723204, "meanTokensPerSec": 121.69, "genTokensPerSec": 121.69, "genTokens": 40876, "promptTokensPerSec": 1665.83, "peakRssMb": 21213, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 306443, "output": 9758 } } }, "usage": { "available": true, "totalInputTokens": 1419591, "totalOutputTokens": 40876, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 17, "todayTokensIn": 306443, "todayTokensCached": 206184, "todayTokensOut": 9758, "todayCost": 0, "totalTurns": 17, "totalTokensIn": 306443, "totalTokensCached": 206184, "totalTokensOut": 9758, "totalCost": 0, "medianOutputTokensPerSec": 123, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 123, "turns": 17 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 37142, "cachedInputTokens": 36157, "outputTokens": 241, "cost": 0, "durationMs": 28502, "outputTokensPerSec": 141.34209026105884, "at": "2026-10-28T15:23:18.957Z" }, "lastUpdated": "2026-10-28T15:23:18.957Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:36:23.572Z" } }, "source": "engine-log" }, "process": { "sampleCount": 311, "peakRssMb": 21213, "peakCpuPercent": 264.1, "firstAtMs": 0, "lastAtMs": 1719628 }, "gpu": { "available": true, "sampleCount": 311, "peakUtilPercent": 100, "peakMemUsedMb": 10519, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 311, "peakUsedMb": 31515, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] orderly-pricing/1 paused for help (gate_exhausted) at step \"reproduce\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/command-tool-availability.json preserves all 10 model-facing rosters with exact daemon line numbers, the first required-run gate note and SHA256 hashes of both source logs.", "Required gate on orderly-pricing/1, step reproduce, at 10/28/2026 15:08:13 explicitly demands run_package_script; zero captured model-facing rosters include it.", "Recorded tool calls contain no package runner; the broad registered bridge roster is not the model-visible filtered tool list." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; the underlying policy layer causing omission is not established. Preserve original model-default result and fixed score, strict planned attempt count, artifact flaws and actual judgments/performance. This supplemental eligibility supersedes earlier provisional no-exclusion notes for this trial. No source, dependency, tool permission, gate or model tuning was changed.", "reviewedAt": "2026-10-28T15:44:56.3819562Z" }, "startedAt": "2026-10-28T15:07:37.018Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T15:45:34.756Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read complete fix/repro/diagnosis/validation/review/blocker artifacts, own source/tests/package/brief and actual captured chronology. Exact frozen Repro-first discipline/Fix minimality/Honest verification axes. Context preserves first test write, syntax-only validation, actual minimal edit, tool counts, required command gate and filtered roster evidence. Missing mandatory execution tool is a separate aggregate capability exclusion, not a fabricated failed test or excuse for false verification claims. No new execution; selected input is untruncated and below 20 KiB.", "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs", "artifactPath": "artifacts/orderly-pricing/tasks/1/fix-notes.md", "contextArtifactPaths": [ "artifacts/orderly-pricing/tasks/1/repro.md", "artifacts/orderly-pricing/tasks/1/diagnosis.md", "artifacts/orderly-pricing/tasks/1/validation.md", "artifacts/orderly-pricing/tasks/1/review.md", "artifacts/orderly-pricing/tasks/1/blocker.md", "workspace/orderly-pricing/source/bug-report.md", "workspace/orderly-pricing/package.json", "workspace/orderly-pricing/src/pricing.js", "workspace/orderly-pricing/tests/pricing.test.mjs", "workspace/orderly-pricing/tests/repro.test.mjs", "judge-context-bugfix.json" ], "artifactKind": "markdown", "scoreAxes": { "Repro-first discipline": 3, "Fix minimality": 8, "Honest verification": 2 }, "justification": "A real regression test precedes the correct one-character boundary fix, with no input special-case. Its quoted red output reverses expected/actual and is not a recorded run. Validation and PASS review claim execution absent from the trace; validate only checked syntax. A later blocker honestly identifies the unavailable runner, but contradicts those claims.", "artifactSha256": "5349607fb0160cf65a15242d9187ae23de18beee12aed5c7afe040472240af9a", "artifactBytes": 978, "contextArtifacts": [ { "path": "artifacts/orderly-pricing/tasks/1/repro.md", "bytes": 1068, "sha256": "ef522aa35bbde80e2fd6166adbfe042c4330d3e53e383b04ec2c34460c30f01d" }, { "path": "artifacts/orderly-pricing/tasks/1/diagnosis.md", "bytes": 1055, "sha256": "7fac5cfc6cc61efa0d3654e74409e28db9c6a5259d8e119e41288a1549a99a75" }, { "path": "artifacts/orderly-pricing/tasks/1/validation.md", "bytes": 471, "sha256": "5d655aa2658e7953d5e5dc72d3196d4e889df3c96b28182b42d7d9d34acad529" }, { "path": "artifacts/orderly-pricing/tasks/1/review.md", "bytes": 410, "sha256": "210705628954d768888d546a33b2d1f549e9cefaca40fbdde031cd14a5b59056" }, { "path": "artifacts/orderly-pricing/tasks/1/blocker.md", "bytes": 813, "sha256": "fed5137ce74fa024be69bf9b8b406babe1fe747cb3495d9d74309a0c42a35d02" }, { "path": "workspace/orderly-pricing/source/bug-report.md", "bytes": 313, "sha256": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f" }, { "path": "workspace/orderly-pricing/package.json", "bytes": 131, "sha256": "0bdb539d11fe5cd1b21c74c3f5e3051d57759f9d63c667d6f4f59bc8ab2f67a7" }, { "path": "workspace/orderly-pricing/src/pricing.js", "bytes": 473, "sha256": "4c03c76d7d58077171a545c26ae25b579dbfede2e12348247be0194d54942126" }, { "path": "workspace/orderly-pricing/tests/pricing.test.mjs", "bytes": 486, "sha256": "046f652bdfde245458ac63c1c3b2e08723a2b656772e1f2b4572da57969d4628" }, { "path": "workspace/orderly-pricing/tests/repro.test.mjs", "bytes": 336, "sha256": "3d4bc8b793b4f12edf374bdd1f7bf2a1c15d986a79624079e70798b9e8f4418b" }, { "path": "judge-context-bugfix.json", "bytes": 5954, "sha256": "520ad665099f5be13018f1b9d468405df5cc5fe86721a2b45537c4764ed4a491" } ], "inputBytes": 12488, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4.333333333333333 }, "systemMemory": { "sampleCount": 311, "peakUsedMb": 31515, "totalMb": 31865 }, "evidenceHashes": { "result.json": "75f9c781b8ad3433214f338da6379004da970b3fec98aae247fb9cc450a361f0", "score.json": "5b54e1bc1ff48b56958318af60ab0a56bbe8985b151d356dac6e515ad873aced", "metrics.json": "7283c7e2cd977dc180115839b5ccb97c813d0f70f25ea47133c31bbe7bf86de4", "codex-inline-judge.json": "67b89e64f86adef19d4eb964b21c020fd6f7ff2b0aba380c62a097b8d939a04c" }, "rubricId": "craftbook-bug-fix-tdd", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on workspace output; missing signals stayed unchanged: task sourced from craftbook bug-fix-tdd has not reached a terminal step. Last failure: task sourced from craftbook bug-fix-tdd has not reached a terminal step" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 30013 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 1068, "score": 27, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not reached a terminal step" } } ] }, "quality": { "score": 9.6, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 96.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 30013 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 1068, "score": 27, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not reached a terminal step" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.19197255555555556 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 233 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 30013 }, { "source": "facts", "path": "outcome.durationMs", "value": 1727753 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 5.6, "band": "needs-tuning" } }, { "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho", "runDir": "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho", "finishedAt": "2026-10-28T15:57:35.414Z", "rawSuccess": false, "rawReason": "craftbook-bug-fix-tdd stale no-write repair loop: tasks/1/review.md had unchanged content (729 bytes) for 122s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 27/28: tasks/1/review.md is missing required content: explicit reviewer verdict — nothing in its 729 bytes matches /Verdict:\\s*(?:PASS|REVISE)/. Add that content.", "durationMs": 1269345, "rawFixedComposite": 5.6, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 4.333333333333333, "judgeArtifactCandidates": [ "artifacts/orderly-pricing/tasks/1/fix-notes.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 119.57, "nativePrefillTokensPerSec": 1646.94, "inputTokens": 793721, "outputTokens": 48834, "peakRssMb": 21015, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 1269345, "derived": { "durationMs": 1260905, "meanTokensPerSec": 119.57, "genTokensPerSec": 119.57, "genTokens": 48834, "promptTokensPerSec": 1646.94, "peakRssMb": 21015, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 469995, "output": 8567 } } }, "usage": { "available": true, "totalInputTokens": 793721, "totalOutputTokens": 48834, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 25, "todayTokensIn": 469995, "todayTokensCached": 437515, "todayTokensOut": 8567, "todayCost": 0, "totalTurns": 25, "totalTokensIn": 469995, "totalTokensCached": 437515, "totalTokensOut": 8567, "totalCost": 0, "medianOutputTokensPerSec": 143.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 143.2, "turns": 25 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8532, "cachedInputTokens": 7700, "outputTokens": 104, "cost": 0, "durationMs": 11945, "outputTokensPerSec": 162.89452500430957, "at": "2026-10-28T15:57:25.388Z" }, "lastUpdated": "2026-10-28T15:57:25.388Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T15:57:30.551Z" } }, "source": "engine-log" }, "process": { "sampleCount": 227, "peakRssMb": 21015, "peakCpuPercent": 266.2, "firstAtMs": 0, "lastAtMs": 1259364 }, "gpu": { "available": true, "sampleCount": 227, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 227, "peakUsedMb": 31273, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] orderly-pricing/1 paused for help (gate_exhausted) at step \"reproduce\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/command-tool-availability.json preserves all 11 model-facing rosters with exact daemon line numbers, first required-run gate and source SHA256 hashes.", "Required reproduce gate at 2026-10-28T15:37:02.709Z explicitly demands run_package_script; none of the captured filtered rosters includes it.", "Complete captured tool-call inventory contains no package runner or test execution; generic registered-bridge availability is not model exposure." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; underlying policy cause remains unestablished. Preserve raw fixed result, strict planned attempt count, actual artifact findings and judging/performance. The separate wrong-location reviewer repair and unsupported green claims remain valid findings. No source, dependency, permission, gate or tuning change.", "reviewedAt": "2026-10-28T16:04:22.645Z" }, "startedAt": "2026-10-28T15:36:26.069Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T16:04:22.740Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read the complete required artifacts, alternate workspace review/task notes, actual brief/source/test and captured chronology. Applied the exact frozen Repro-first discipline/Fix minimality/Honest verification axes. The regression was written before the fix, but validate checked parsing only and the trace contains no test execution. Correct persisted code and misleading verification claims are judged separately from the independently confirmed mandatory-runner exclusion. No new execution or rendering; all selected inputs are untruncated under 20 KiB.", "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho", "artifactPath": "artifacts/orderly-pricing/tasks/1/fix-notes.md", "contextArtifactPaths": [ "artifacts/orderly-pricing/tasks/1/repro.md", "artifacts/orderly-pricing/tasks/1/diagnosis.md", "artifacts/orderly-pricing/tasks/1/validation.md", "artifacts/orderly-pricing/tasks/1/review.md", "artifacts/orderly-pricing/tasks/1/notes.md", "artifacts/orderly-pricing/tasks/1/task-notes.md", "workspace/orderly-pricing/source/bug-report.md", "workspace/orderly-pricing/package.json", "workspace/orderly-pricing/src/pricing.js", "workspace/orderly-pricing/tests/pricing.test.mjs", "workspace/orderly-pricing/tasks/1/review.md", "workspace/orderly-pricing/task-notes.md", "judge-context-bugfix.json" ], "artifactKind": "markdown", "scoreAxes": { "Repro-first discipline": 3, "Fix minimality": 8, "Honest verification": 2 }, "justification": "A genuine boundary regression precedes the correct one-character fix, but quoted red output is not a recorded test run. Validation calls execution simulated yet asserts a green suite; the review accepts that unsupported claim. Notes admit the missing runner. Repeated repairs target an alternate workspace review, leaving the required artifact unchanged.", "artifactSha256": "31b1aa8bcfa74dc172aa59bfd183e876232e33c4cda7ec28f680bc3465158087", "artifactBytes": 869, "contextArtifacts": [ { "path": "artifacts/orderly-pricing/tasks/1/repro.md", "bytes": 903, "sha256": "796e6413cfdc540452a7b90a6eba4267d9c0d4898b0340bae1ab9aa500c0192f" }, { "path": "artifacts/orderly-pricing/tasks/1/diagnosis.md", "bytes": 745, "sha256": "1a5ac2607119f438f1ce1c151046bcb6e4240c3c7c6efbe6a5dcb2e4a2ebe021" }, { "path": "artifacts/orderly-pricing/tasks/1/validation.md", "bytes": 653, "sha256": "581a4e9ddcacd0f92b5f07680e1188363a39766cbc8d3f9b026405d10ecaa547" }, { "path": "artifacts/orderly-pricing/tasks/1/review.md", "bytes": 729, "sha256": "16d71020be8050b14725df3c0bb5a460935790360d3ec63e02b1eac2530c7495" }, { "path": "artifacts/orderly-pricing/tasks/1/notes.md", "bytes": 1048, "sha256": "22d443ee8a9074279ec308cd11081fbbef296c648c7912bb7e2705ed82df8d8c" }, { "path": "artifacts/orderly-pricing/tasks/1/task-notes.md", "bytes": 382, "sha256": "5c7ca230bf2750493fedf762a28a6f7010859924e02dd6986f60e2dcf1aec5db" }, { "path": "workspace/orderly-pricing/source/bug-report.md", "bytes": 313, "sha256": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f" }, { "path": "workspace/orderly-pricing/package.json", "bytes": 131, "sha256": "0bdb539d11fe5cd1b21c74c3f5e3051d57759f9d63c667d6f4f59bc8ab2f67a7" }, { "path": "workspace/orderly-pricing/src/pricing.js", "bytes": 473, "sha256": "4c03c76d7d58077171a545c26ae25b579dbfede2e12348247be0194d54942126" }, { "path": "workspace/orderly-pricing/tests/pricing.test.mjs", "bytes": 547, "sha256": "1e7c2acd8303ebea677b1d1780bcf21ee0c2098c29afae354f7f137f36721c7f" }, { "path": "workspace/orderly-pricing/tasks/1/review.md", "bytes": 364, "sha256": "ca3b2ea3d5963a1e0b923a7abfc2f97db1bb8fdf5007dfc4c297bf9806da28dd" }, { "path": "workspace/orderly-pricing/task-notes.md", "bytes": 383, "sha256": "38edb3251c1a69b02be15bee03f852c7fb30ce383cd9dc387074384a91643673" }, { "path": "judge-context-bugfix.json", "bytes": 8988, "sha256": "285b7c80b5ec84594dff1719a2430c6773dd1853d740b1cd9e4ba66f45fa40c4" } ], "inputBytes": 16528, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4.333333333333333 }, "systemMemory": { "sampleCount": 227, "peakUsedMb": 31273, "totalMb": 31865 }, "evidenceHashes": { "result.json": "6adbecaba67fefb9da0c2b93e6cc54a04e4dfaf958330f590aed9eddbc476af4", "score.json": "c10254d29e7703adc0af43b566816df4f9fa32245451a4a618ff762de411905b", "metrics.json": "71845eabdf3b8342266b02a312acc97d456faeb6638f5267fc643f49ae5fbadb", "codex-inline-judge.json": "a00030b6f48d2e7c14b5d5885a3672b616e82c1bb78b040f864034130154fac8" }, "rubricId": "craftbook-bug-fix-tdd", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "craftbook-bug-fix-tdd stale no-write repair loop: tasks/1/review.md had unchanged content (729 bytes) for 122s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 27/28: tasks/1/review.md is missing required content: explicit reviewer verdict — nothing in its 729 bytes matches /Verdict:\\s*(?:PASS|REVISE)/. Add that content." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 29131 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 903, "score": 27, "scoreMax": 28, "signals": [], "failReason": "tasks/1/review.md is missing required content: explicit reviewer verdict — nothing in its 729 bytes matches /Verdict:\\s*(?:PASS|REVISE)/. Add that content." } } ] }, "quality": { "score": 9.6, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 96.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 29131 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 903, "score": 27, "scoreMax": 28, "signals": [], "failReason": "tasks/1/review.md is missing required content: explicit reviewer verdict — nothing in its 729 bytes matches /Verdict:\\s*(?:PASS|REVISE)/. Add that content." } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.14103833333333332 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 242 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 29131 }, { "source": "facts", "path": "outcome.durationMs", "value": 1269345 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.6, "band": "needs-tuning" } }, { "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic", "runDir": "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic", "finishedAt": "2026-10-28T16:41:49.834Z", "rawSuccess": false, "rawReason": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 26 while the failing detail kept changing; latest checked file workspace output. Last failure: task sourced from craftbook bug-fix-tdd has not reached a terminal step", "durationMs": 2653762, "rawFixedComposite": 5.5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 4, "judgeArtifactCandidates": [ "artifacts/orderly-pricing/tasks/1/fix-notes.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 114.86, "nativePrefillTokensPerSec": 1415.67, "inputTokens": 1537944, "outputTokens": 27830, "peakRssMb": 21287, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 2653762, "derived": { "durationMs": 2648839, "meanTokensPerSec": 114.86, "genTokensPerSec": 114.86, "genTokens": 27830, "promptTokensPerSec": 1415.67, "peakRssMb": 21287, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 178858, "output": 5749 } } }, "usage": { "available": true, "totalInputTokens": 1537944, "totalOutputTokens": 27830, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 178858, "todayTokensCached": 134081, "todayTokensOut": 5749, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 178858, "totalTokensCached": 134081, "totalTokensOut": 5749, "totalCost": 0, "medianOutputTokensPerSec": 148, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 148, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 19559, "cachedInputTokens": 19506, "outputTokens": 299, "cost": 0, "durationMs": 19112, "outputTokensPerSec": 147.99665666613197, "at": "2026-10-28T16:03:22.185Z" }, "lastUpdated": "2026-10-28T16:03:22.185Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:41:48.620Z" } }, "source": "engine-log" }, "process": { "sampleCount": 479, "peakRssMb": 21287, "peakCpuPercent": 263.2, "firstAtMs": 0, "lastAtMs": 2646280 }, "gpu": { "available": true, "sampleCount": 479, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 479, "peakUsedMb": 30930, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "task-paused-unrecovered", "failureClassEvidence": "[tasks] orderly-pricing/1 paused for help (gate_exhausted) at step \"reproduce\"", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic", "status": "excluded", "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface.", "rawFailureClass": "model", "rawFailureClassRule": "task-paused-unrecovered", "evidence": [ "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/command-tool-availability.json preserves all three filtered rosters with line numbers, required gate and source hashes.", "Required reproduce gate at 2026-10-28T15:58:22.874Z demands the missing runner.", "Complete captured tool inventory contains no test runner; an actual ask_user_question at 15:58:55 identifies the missing tool." ], "diagnosis": "Confirmed execution-surface/gate contract mismatch; underlying policy cause unestablished. Preserve raw outcome, fixed score, strict attempt count, artifact judgment and performance. Unsupported final execution claims remain separate findings. No runtime/source/permission/tuning/gate change.", "reviewedAt": "2026-10-28T16:53:40.105Z" }, "startedAt": "2026-10-28T15:57:36.072Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "eval-setup-command-tool-availability", "reason": "The live task gate requires a step-attributed npm run test receipt, but every captured model-facing roster omits run_package_script. The measured workflow cannot complete its mandatory command-evidence gate through the supplied tool surface." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T16:54:03.045Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read complete final reports, own source/test/brief/package, and captured test-write, syntax-validation, missing-tool question and exact source-edit events. Applied frozen exact Repro-first discipline/Fix minimality/Honest verification axes. The unavailable mandatory runner is independently excluded from capability aggregation; the actual code credit and unsupported final execution claims remain judged. No new execution or rendering. Selected inputs are untruncated under 20 KiB.", "suiteId": "developer", "scenarioId": "craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic", "artifactPath": "artifacts/orderly-pricing/tasks/1/fix-notes.md", "contextArtifactPaths": [ "artifacts/orderly-pricing/tasks/1/repro.md", "artifacts/orderly-pricing/tasks/1/diagnosis.md", "artifacts/orderly-pricing/tasks/1/validation.md", "artifacts/orderly-pricing/tasks/1/review.md", "artifacts/orderly-pricing/tasks/1/task-notes.md", "workspace/orderly-pricing/source/bug-report.md", "workspace/orderly-pricing/package.json", "workspace/orderly-pricing/src/pricing.js", "workspace/orderly-pricing/tests/pricing.test.mjs", "workspace/orderly-pricing/tests/repro.test.mjs", "judge-context-bugfix.json" ], "artifactKind": "markdown", "scoreAxes": { "Repro-first discipline": 3, "Fix minimality": 8, "Honest verification": 1 }, "justification": "The real regression test precedes a correct one-character boundary fix with no unrelated source change. No test execution backs the red/green claims: validate only parsed JavaScript, and the model explicitly reported its missing runner. Final validation invents Running 1/3 tests... OK output, while the PASS review treats it as proof.", "artifactSha256": "560d215960b60275020b0090ab11cf6a8922204daf444802071402435393eb9c", "artifactBytes": 1698, "contextArtifacts": [ { "path": "artifacts/orderly-pricing/tasks/1/repro.md", "bytes": 788, "sha256": "370754b5cd8083b33bdf10e4708e4ab4a34db5fc9058e6ea1897463d783a3968" }, { "path": "artifacts/orderly-pricing/tasks/1/diagnosis.md", "bytes": 1301, "sha256": "05ff488724368de52b75ef54d9369939cb0194106ee746925cff1b12fdd5e661" }, { "path": "artifacts/orderly-pricing/tasks/1/validation.md", "bytes": 480, "sha256": "ae5521a8c89f0ff695ddd167300a8a125465805a20e8badc4c92d42e488af037" }, { "path": "artifacts/orderly-pricing/tasks/1/review.md", "bytes": 703, "sha256": "528364fd240de005a23627c792cc6cfb8e15fc4c353954121fe686b49dec79f7" }, { "path": "artifacts/orderly-pricing/tasks/1/task-notes.md", "bytes": 442, "sha256": "436e7a070d9de7b168641edb8cc4f544c8b17b7bf091ce6ac6705f75d94e616d" }, { "path": "workspace/orderly-pricing/source/bug-report.md", "bytes": 313, "sha256": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f" }, { "path": "workspace/orderly-pricing/package.json", "bytes": 131, "sha256": "0bdb539d11fe5cd1b21c74c3f5e3051d57759f9d63c667d6f4f59bc8ab2f67a7" }, { "path": "workspace/orderly-pricing/src/pricing.js", "bytes": 474, "sha256": "fad23b0c7929def833f4dcb9c3557a77ccb3f3ad92b3be97e462fea4e0a56921" }, { "path": "workspace/orderly-pricing/tests/pricing.test.mjs", "bytes": 389, "sha256": "e0fdd0f6a038c279dbfcf2504cff15728b35d51e4c22645fc6dd6aedf3f6788f" }, { "path": "workspace/orderly-pricing/tests/repro.test.mjs", "bytes": 272, "sha256": "ebbaf63446c01b18a8b70777f18ac0c585f86d3d30d455b6b98a442781e94e87" }, { "path": "judge-context-bugfix.json", "bytes": 7146, "sha256": "b98a8e6234721bb4ae9dd4b7ba0f0bf8a67e9d26f003b2bbe3f860eb800a1f25" } ], "inputBytes": 14137, "inputLimitBytes": 20480, "truncated": false, "meanScore": 4 }, "systemMemory": { "sampleCount": 479, "peakUsedMb": 30930, "totalMb": 31865 }, "evidenceHashes": { "result.json": "56a0c98d5502380584eb5958730bc736f07b4bf36f6cbd122cdf01e1e73df2c9", "score.json": "a422f467cbfc45f6d69f6754889617cfd5f88e0dbf79de36e082eb0cccd013cf", "metrics.json": "068b11a9e933f341714d4bacc6fd0e983435327f207909038d8a06b6d9df0b7d", "codex-inline-judge.json": "ec349247515c82cb1c040c545282f89ba2e39757183d474a49eecc9ff1f6570b" }, "rubricId": "craftbook-bug-fix-tdd", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "task-paused-unrecovered", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 26 while the failing detail kept changing; latest checked file workspace output. Last failure: task sourced from craftbook bug-fix-tdd has not reached a terminal step" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 41497 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 788, "score": 26, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not reached a terminal step" } } ] }, "quality": { "score": 9.3, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 92.9% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 41497 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 788, "score": 26, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not reached a terminal step" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2948624444444444 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 182 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 41497 }, { "source": "facts", "path": "outcome.durationMs", "value": 2653762 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 5.5, "band": "needs-tuning" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248", "runDir": "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248", "finishedAt": "2026-10-28T16:43:35.230Z", "rawSuccess": true, "rawReason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "durationMs": 97586, "rawFixedComposite": 8.8, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [ "workspace/shop-site-pages/pages/index.html", "workspace/shop-site-pages/pages/shipping.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 104.09, "nativePrefillTokensPerSec": 1855.09, "inputTokens": 34847, "outputTokens": 4386, "peakRssMb": 16870, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 97586, "derived": { "durationMs": 90195, "meanTokensPerSec": 104.09, "genTokensPerSec": 104.09, "genTokens": 4386, "promptTokensPerSec": 1855.09, "peakRssMb": 16870, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 12120, "output": 16 } } }, "usage": { "available": true, "totalInputTokens": 34847, "totalOutputTokens": 4386, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12120, "todayTokensCached": 11888, "todayTokensOut": 16, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12120, "totalTokensCached": 11888, "totalTokensOut": 16, "totalCost": 0, "medianOutputTokensPerSec": 109.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 109.2, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12120, "cachedInputTokens": 11888, "outputTokens": 16, "cost": 0, "durationMs": 31269, "outputTokensPerSec": 109.1830198567519, "at": "2026-10-28T16:42:34.280Z" }, "lastUpdated": "2026-10-28T16:42:34.280Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:43:30.803Z" } }, "source": "engine-log" }, "process": { "sampleCount": 17, "peakRssMb": 16870, "peakCpuPercent": 269.3, "firstAtMs": 4, "lastAtMs": 89118 }, "gpu": { "available": true, "sampleCount": 17, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 27728, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T16:41:57.644Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T16:54:42.300Z", "suiteId": "complex-work", "scenarioId": "craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "This selection-only scenario has no declared qualitative rubric. Both HTML candidates were read and verified byte-for-byte against seeded PAGE_FIXTURES; captured model tools contain no write/replace/append. The task ends at choosing/invoking the bundled recipe, so these unchanged inputs are not model-produced HTML deliverables for the legacy fallback rubric. Deterministic selection success and performance remain recorded; no qualitative score was invented.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null, "evidence": { "fixtureSourcePath": "evals/src/craftbooks/authoring/find-vs-create.ts", "fixtureSourceSha256": "2c87ca556da08b658e27649cdac666be492ce334df9d09116b08c2c22a998916", "recordingSha256": "d8a5794014b86a917a4eeda6b13e36a9c6b4123c4f8cef7f4c4437bbdb790066", "toolCounts": { "suggest_work": 2, "search": 1, "start_work": 1, "list_artifacts": 1, "suggest_craftbook": 1, "read_task_notes": 1, "invoke_craftbook": 1 }, "fixtures": [ { "path": "workspace/shop-site-pages/pages/index.html", "bytes": 168, "sha256": "247f7de92e8ca6a36b9e7c485e7b7649efd85739ef211faa04c946324f284fb8", "matchesSeedExactly": true }, { "path": "workspace/shop-site-pages/pages/shipping.html", "bytes": 165, "sha256": "1ce475ec4ca411c79d190c428b288947732ec62344799d746155e08ac208ef8b", "matchesSeedExactly": true } ] } }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 27728, "totalMb": 31865 }, "evidenceHashes": { "result.json": "f4c5e510ad88f00a82f8c0c847bbd411532750eb0172afd03f524fa1ecbbc377", "score.json": "4177cf94247be8c297ff7a8e589bb923aef0e5e975638125ad700f71ab6a305b", "metrics.json": "afa67bd281073e3b3360bd533de5df44850feaa517a365dd47635438a6a7e778", "codex-inline-judge.json": "7e2eed02eb65e2538c6ac5995754239355dadffdf99312394d4512f7364d58bf" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248", "scenarioId": "craftbook-find-vs-create", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08132166666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 97586 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt", "runDir": "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt", "finishedAt": "2026-10-28T16:44:24.373Z", "rawSuccess": true, "rawReason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "durationMs": 48152, "rawFixedComposite": 8.8, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [ "workspace/shop-site-pages/pages/index.html", "workspace/shop-site-pages/pages/shipping.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 104.14, "nativePrefillTokensPerSec": 1868.81, "inputTokens": 11954, "outputTokens": 2260, "peakRssMb": 12187, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 48152, "derived": { "durationMs": 39766, "meanTokensPerSec": 104.14, "genTokensPerSec": 104.14, "genTokens": 2260, "promptTokensPerSec": 1868.81, "peakRssMb": 12187, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 11938, "output": 43 } } }, "usage": { "available": true, "totalInputTokens": 11954, "totalOutputTokens": 2260, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 11938, "todayTokensCached": 11772, "todayTokensOut": 43, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 11938, "totalTokensCached": 11772, "totalTokensOut": 43, "totalCost": 0, "medianOutputTokensPerSec": 93.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 93.9, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 11938, "cachedInputTokens": 11772, "outputTokens": 43, "cost": 0, "durationMs": 36407, "outputTokensPerSec": 93.93598486288701, "at": "2026-10-28T16:44:18.279Z" }, "lastUpdated": "2026-10-28T16:44:18.279Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:44:19.260Z" } }, "source": "engine-log" }, "process": { "sampleCount": 8, "peakRssMb": 12187, "peakCpuPercent": 267.7, "firstAtMs": 1, "lastAtMs": 38674 }, "gpu": { "available": true, "sampleCount": 8, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 22715, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T16:43:36.221Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T16:54:42.307Z", "suiteId": "complex-work", "scenarioId": "craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "This selection-only scenario has no declared qualitative rubric. Both HTML candidates were read and verified byte-for-byte against seeded PAGE_FIXTURES; captured model tools contain no write/replace/append. The task ends at choosing/invoking the bundled recipe, so these unchanged inputs are not model-produced HTML deliverables for the legacy fallback rubric. Deterministic selection success and performance remain recorded; no qualitative score was invented.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null, "evidence": { "fixtureSourcePath": "evals/src/craftbooks/authoring/find-vs-create.ts", "fixtureSourceSha256": "2c87ca556da08b658e27649cdac666be492ce334df9d09116b08c2c22a998916", "recordingSha256": "6dba911618ebe3ae4b1b44ed12c1f8000cf9e82cd7ac3a1894ebbdd180adbc4c", "toolCounts": { "suggest_work": 2, "search": 1, "start_work": 1 }, "fixtures": [ { "path": "workspace/shop-site-pages/pages/index.html", "bytes": 168, "sha256": "247f7de92e8ca6a36b9e7c485e7b7649efd85739ef211faa04c946324f284fb8", "matchesSeedExactly": true }, { "path": "workspace/shop-site-pages/pages/shipping.html", "bytes": 165, "sha256": "1ce475ec4ca411c79d190c428b288947732ec62344799d746155e08ac208ef8b", "matchesSeedExactly": true } ] } }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 22715, "totalMb": 31865 }, "evidenceHashes": { "result.json": "338cdc3a57ad0006bea621f8d9784e3e90d553bd112a03663d1f74d4265f788b", "score.json": "5d9b1ef20d422c95d230311ff0a1c6e8de4e78053da91f0e5dd77a7a49d900db", "metrics.json": "be4bf01fed5944eea184f3f240d3196e2a6c8c25d3445081e6664027ae73f64e", "codex-inline-judge.json": "fb78cc1d91c10434b41388583c7f1e36dceb0c9e41b647f47b8c661d9bf0dda1" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt", "scenarioId": "craftbook-find-vs-create", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.040126666666666665 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 48152 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese", "runDir": "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese", "finishedAt": "2026-10-28T16:45:14.534Z", "rawSuccess": true, "rawReason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "durationMs": 49091, "rawFixedComposite": 8.8, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [ "workspace/shop-site-pages/pages/index.html", "workspace/shop-site-pages/pages/shipping.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 108.41, "nativePrefillTokensPerSec": 1867.13, "inputTokens": 12183, "outputTokens": 2121, "peakRssMb": 13144, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 49091, "derived": { "durationMs": 39715, "meanTokensPerSec": 108.41, "genTokensPerSec": 108.41, "genTokens": 2121, "promptTokensPerSec": 1867.13, "peakRssMb": 13144, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 12167, "output": 56 } } }, "usage": { "available": true, "totalInputTokens": 12183, "totalOutputTokens": 2121, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12167, "todayTokensCached": 11991, "todayTokensOut": 56, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12167, "totalTokensCached": 11991, "totalTokensOut": 56, "totalCost": 0, "medianOutputTokensPerSec": 88.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 88.3, "turns": 1 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 12167, "cachedInputTokens": 11991, "outputTokens": 56, "cost": 0, "durationMs": 34243, "outputTokensPerSec": 88.30022075055187, "at": "2026-10-28T16:45:04.905Z" }, "lastUpdated": "2026-10-28T16:45:04.905Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:45:08.011Z" } }, "source": "engine-log" }, "process": { "sampleCount": 8, "peakRssMb": 13144, "peakCpuPercent": 264.6, "firstAtMs": 3, "lastAtMs": 38660 }, "gpu": { "available": true, "sampleCount": 8, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 23693, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T16:44:25.443Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T16:54:42.313Z", "suiteId": "complex-work", "scenarioId": "craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "This selection-only scenario has no declared qualitative rubric. Both HTML candidates were read and verified byte-for-byte against seeded PAGE_FIXTURES; captured model tools contain no write/replace/append. The task ends at choosing/invoking the bundled recipe, so these unchanged inputs are not model-produced HTML deliverables for the legacy fallback rubric. Deterministic selection success and performance remain recorded; no qualitative score was invented.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null, "evidence": { "fixtureSourcePath": "evals/src/craftbooks/authoring/find-vs-create.ts", "fixtureSourceSha256": "2c87ca556da08b658e27649cdac666be492ce334df9d09116b08c2c22a998916", "recordingSha256": "aa1653fbc539842c39dffb31d904c8b2ede936830954bdf46a9bc048d6479f4c", "toolCounts": { "suggest_work": 2, "search": 1, "start_work": 1 }, "fixtures": [ { "path": "workspace/shop-site-pages/pages/index.html", "bytes": 168, "sha256": "247f7de92e8ca6a36b9e7c485e7b7649efd85739ef211faa04c946324f284fb8", "matchesSeedExactly": true }, { "path": "workspace/shop-site-pages/pages/shipping.html", "bytes": 165, "sha256": "1ce475ec4ca411c79d190c428b288947732ec62344799d746155e08ac208ef8b", "matchesSeedExactly": true } ] } }, "systemMemory": { "sampleCount": 8, "peakUsedMb": 23693, "totalMb": 31865 }, "evidenceHashes": { "result.json": "0b73d508005d85f3fb0c9e833de8a5753138574ce15096b15c4f43821b9a6dfc", "score.json": "1e2e0074e503765ecd1ddd17d809fee1a02a062b98c461c412da6a0a112821df", "metrics.json": "d3866137979cc3b8c152982be5dedbb804d513f853e04cc5dbb6cc4b07cd19fd", "codex-inline-judge.json": "58c1c89fc117c51d0f1cf590d78352a908cc2f59583586c5b4cc469f13ade13a" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese", "scenarioId": "craftbook-find-vs-create", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.04090916666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 49091 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i", "runDir": "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i", "finishedAt": "2026-10-28T16:47:56.009Z", "rawSuccess": true, "rawReason": "diagnosed the unwinnable gate, rewrote the failing step prompt, grew the recipe with a verification step, and completed the task", "durationMs": 156924, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 105.03, "nativePrefillTokensPerSec": 1728.8, "inputTokens": 70838, "outputTokens": 6027, "peakRssMb": 18422, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 156924, "derived": { "durationMs": 150991, "meanTokensPerSec": 105.03, "genTokensPerSec": 105.03, "genTokens": 6027, "promptTokensPerSec": 1728.8, "peakRssMb": 18422, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 93052, "output": 1160 } } }, "usage": { "available": true, "totalInputTokens": 70838, "totalOutputTokens": 6027, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 93052, "todayTokensCached": 86896, "todayTokensOut": 1160, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 93052, "totalTokensCached": 86896, "totalTokensOut": 1160, "totalCost": 0, "medianOutputTokensPerSec": 100.6, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 100.6, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15474, "cachedInputTokens": 15347, "outputTokens": 30, "cost": 0, "durationMs": 18220, "outputTokensPerSec": 125.90849491590183, "at": "2026-10-28T16:47:49.156Z" }, "lastUpdated": "2026-10-28T16:47:49.156Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:47:53.087Z" } }, "source": "engine-log" }, "process": { "sampleCount": 28, "peakRssMb": 18422, "peakCpuPercent": 291.2, "firstAtMs": 1, "lastAtMs": 149831 }, "gpu": { "available": true, "sampleCount": 28, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 29194, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T16:45:19.085Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T16:52:04.060Z", "suiteId": "complex-work", "scenarioId": "craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 28, "peakUsedMb": 29194, "totalMb": 31865 }, "evidenceHashes": { "result.json": "75f5fd89f5f6143022b46d0c9c9af101a2c3fd8b1c691270648f9421cfa6b9d1", "score.json": "458935aee0e2dd53c54bf83be1ffadf6d3b02d7440223b8ebe3df5a29a7cb3c0", "metrics.json": "78fa2fbd482d8cb9b2fa5ee7d43c40a1d073d082fb4b76f88ed67709bdbabcca", "codex-inline-judge.json": "edc43bd4dbb9c7edfe7c0885d8e71c322617296e199d50928d10a8eb9cc1b606" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i", "scenarioId": "craftbook-edit-midtask", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 118420 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-edit-midtask", "bytes": 1247, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.05812 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 31 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 118420 }, { "source": "facts", "path": "outcome.durationMs", "value": 156924 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig", "runDir": "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig", "finishedAt": "2026-10-28T16:50:49.285Z", "rawSuccess": true, "rawReason": "diagnosed the unwinnable gate, rewrote the failing step prompt, grew the recipe with a verification step, and completed the task", "durationMs": 172467, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 106.42, "nativePrefillTokensPerSec": 1775.09, "inputTokens": 85137, "outputTokens": 7024, "peakRssMb": 21110, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 172467, "derived": { "durationMs": 161910, "meanTokensPerSec": 106.42, "genTokensPerSec": 106.42, "genTokens": 7024, "promptTokensPerSec": 1775.09, "peakRssMb": 21110, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 101583, "output": 898 } } }, "usage": { "available": true, "totalInputTokens": 85137, "totalOutputTokens": 7024, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 101583, "todayTokensCached": 60656, "todayTokensOut": 898, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 101583, "totalTokensCached": 60656, "totalTokensOut": 898, "totalCost": 0, "medianOutputTokensPerSec": 103.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 103.4, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7993, "cachedInputTokens": 7745, "outputTokens": 30, "cost": 0, "durationMs": 14949, "outputTokensPerSec": 134.63574068227823, "at": "2026-10-28T16:50:41.363Z" }, "lastUpdated": "2026-10-28T16:50:41.363Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:50:42.176Z" } }, "source": "engine-log" }, "process": { "sampleCount": 30, "peakRssMb": 21110, "peakCpuPercent": 264.2, "firstAtMs": 1, "lastAtMs": 160867 }, "gpu": { "available": true, "sampleCount": 30, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 31180, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T16:47:56.818Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T16:52:04.076Z", "suiteId": "complex-work", "scenarioId": "craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 31180, "totalMb": 31865 }, "evidenceHashes": { "result.json": "4623fdbf81f3800171c312459ea92d1d526802aa4065079c0b854e6a46b93f95", "score.json": "dff63797cdc3d81d23b937de11e5e0423d0f31191da9986f4e8cc3b775ab4472", "metrics.json": "38bb4b0a2183da41db169a36cadca7f2fe0958fda825e0a1b58a6b7963aac1db", "codex-inline-judge.json": "a4ab32652036ba3c841f666caadfed13d52091c80f472960a19830594d61a94b" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig", "scenarioId": "craftbook-edit-midtask", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 89441 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-edit-midtask", "bytes": 1021, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06387666666666666 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 31 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 89441 }, { "source": "facts", "path": "outcome.durationMs", "value": 172467 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv", "runDir": "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv", "finishedAt": "2026-10-28T16:58:13.428Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on task quarterly-brief/1 craftbook; missing signals stayed unchanged: task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it, task quarterly-brief/1's craftbook still has 2 steps — add a verification step at the end of the recipe. Last failure: task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it", "durationMs": 443438, "rawFixedComposite": 4.5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 111.83, "nativePrefillTokensPerSec": 1731.36, "inputTokens": 203156, "outputTokens": 24621, "peakRssMb": 20103, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 443438, "derived": { "durationMs": 434949, "meanTokensPerSec": 111.83, "genTokensPerSec": 111.83, "genTokens": 24621, "promptTokensPerSec": 1731.36, "peakRssMb": 20103, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 345982, "output": 9535 } } }, "usage": { "available": true, "totalInputTokens": 203156, "totalOutputTokens": 24621, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 20, "todayTokensIn": 345982, "todayTokensCached": 292443, "todayTokensOut": 9535, "todayCost": 0, "totalTurns": 20, "totalTokensIn": 345982, "totalTokensCached": 292443, "totalTokensOut": 9535, "totalCost": 0, "medianOutputTokensPerSec": 94.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 94.8, "turns": 20 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 26443, "cachedInputTokens": 26128, "outputTokens": 43, "cost": 0, "durationMs": 35405, "outputTokensPerSec": 91.22898475172683, "at": "2026-10-28T16:58:04.008Z" }, "lastUpdated": "2026-10-28T16:58:04.008Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T16:58:08.417Z" } }, "source": "engine-log" }, "process": { "sampleCount": 79, "peakRssMb": 20103, "peakCpuPercent": 276.3, "firstAtMs": 1, "lastAtMs": 433926 }, "gpu": { "available": true, "sampleCount": 79, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 79, "peakUsedMb": 30242, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T16:50:49.990Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T16:58:58.050Z", "suiteId": "complex-work", "scenarioId": "craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 79, "peakUsedMb": 30242, "totalMb": 31865 }, "evidenceHashes": { "result.json": "ae8494aa9efa5e5da947df0032cacceeb9e71e530ed2ce35994dc0e58df511a7", "score.json": "4595b7b63b89d6b4584653422425e37c009bf1066bf514fe596a6da48479f13c", "metrics.json": "7428aff91671dcead1a9008b53c5a2227ea9714f0a6b25834222703580c6b0e8", "codex-inline-judge.json": "bc87665c0bb98c623d5bda8bc4538c9aa0eb420b46641c487547b28d05483cad" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv", "scenarioId": "craftbook-edit-midtask", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on task quarterly-brief/1 craftbook; missing signals stayed unchanged: task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it, task quarterly-brief/1's craftbook still has 2 steps — add a verification step at the end of the recipe. Last failure: task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 161018 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-edit-midtask", "bytes": 733, "score": 2, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" } } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 161018 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-edit-midtask", "bytes": 733, "score": 2, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.1642362962962963 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 59 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 161018 }, { "source": "facts", "path": "outcome.durationMs", "value": 443438 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.5, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-invoice-run", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1", "runDir": "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1", "finishedAt": "2026-10-28T17:01:16.904Z", "rawSuccess": true, "rawReason": "craftbook-invoice-run passed 10 deterministic craftbook checks", "durationMs": 179514, "rawFixedComposite": 9, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 5.6, "judgeArtifactCandidates": [ "workspace/fieldnote-studio-office/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 139.54, "nativePrefillTokensPerSec": 1865.9, "inputTokens": 87679, "outputTokens": 8630, "peakRssMb": 21359, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 179514, "derived": { "durationMs": 167812, "meanTokensPerSec": 139.54, "genTokensPerSec": 139.54, "genTokens": 8630, "promptTokensPerSec": 1865.9, "peakRssMb": 21359, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 72985, "output": 805 } } }, "usage": { "available": true, "totalInputTokens": 87679, "totalOutputTokens": 8630, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 72985, "todayTokensCached": 68041, "todayTokensOut": 805, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 72985, "totalTokensCached": 68041, "totalTokensOut": 805, "totalCost": 0, "medianOutputTokensPerSec": 136.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 136.5, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16721, "cachedInputTokens": 16418, "outputTokens": 406, "cost": 0, "durationMs": 34050, "outputTokensPerSec": 155.0282553349816, "at": "2026-10-28T17:01:06.326Z" }, "lastUpdated": "2026-10-28T17:01:06.326Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:01:08.877Z" } }, "source": "engine-log" }, "process": { "sampleCount": 31, "peakRssMb": 21359, "peakCpuPercent": 265.4, "firstAtMs": 1, "lastAtMs": 166781 }, "gpu": { "available": true, "sampleCount": 31, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 31, "peakUsedMb": 30924, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T16:58:17.390Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T17:13:29.585Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read complete required report, all three invoice HTML sources, own client roster/ledger, scope/billables/fanout/verdict and captured mutation inventory. Applied all five exact frozen invoice axes. Amounts sum to 5415 and numbering is 042-044, but source ledger still marks work uninvoiced with last number 041. Due dates have no stated issue-date basis and do not establish net-30/net-14 compliance. Print-polish is source/CSS assessment only; no browser or rendered-page claim. Combined hashed inputs are untruncated below 20 KiB.", "suiteId": "complex-work", "scenarioId": "craftbook-invoice-run", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1", "artifactPath": "workspace/fieldnote-studio-office/report.md", "contextArtifactPaths": [ "workspace/fieldnote-studio-office/clients.json", "workspace/fieldnote-studio-office/ledger.json", "workspace/fieldnote-studio-office/invoices/2026-042.html", "workspace/fieldnote-studio-office/invoices/2026-043.html", "workspace/fieldnote-studio-office/invoices/2026-044.html", "artifacts/fieldnote-studio-office/tasks/1/scope.md", "artifacts/fieldnote-studio-office/tasks/1/billables.json", "artifacts/fieldnote-studio-office/tasks/1/draft.md", "artifacts/fieldnote-studio-office/tasks/1/verdict.md", "judge-context-invoice.json" ], "artifactKind": "markdown", "scoreAxes": { "numeric integrity": 7, "numbering continuity": 8, "professionalism": 3, "completeness": 4, "print-polish": 6 }, "justification": "Amounts and 042-044 numbering match the billable ledger; the idle client is explicitly skipped. Books remain unchanged. Due dates lack an issue-date basis, known contacts are replaced by placeholders, and zero-tax claims are unsupported. All three pages have print margins and clear tables, but the reviewer checks dates against its own billables instead of client terms.", "artifactSha256": "e1c3045995bbad0ae0175843b4a86b28d0539e1ccbdcd1cd23523a0ecd081c98", "artifactBytes": 557, "contextArtifacts": [ { "path": "workspace/fieldnote-studio-office/clients.json", "bytes": 579, "sha256": "1aa5d375dd5ecaaf2470f8a402b6b57c8a1c53508f0b49dccaf9baebf8d63170" }, { "path": "workspace/fieldnote-studio-office/ledger.json", "bytes": 878, "sha256": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-042.html", "bytes": 4379, "sha256": "00fe29850ba09322bdeecd8087ca78367638f498101d98e01060f15ffc4bff3b" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-043.html", "bytes": 4622, "sha256": "2eca4354805c9fdce3c603096e7649a4ba553163b38031cfdb8c8912f6f7a4a0" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-044.html", "bytes": 4346, "sha256": "5e248663bb6904a9008c3820be87d746d9436a19a228aba7ca86fdd2727445ee" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/scope.md", "bytes": 339, "sha256": "ca89f0d717bd93d22cc55c0ece845f58b606239724783c4c867e782e6a7d45e1" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/billables.json", "bytes": 538, "sha256": "125dccb2a75bd46a3f9bb54ebff1ac2cfbe5f5021326ad332fc00474915f5a2b" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/draft.md", "bytes": 248, "sha256": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/verdict.md", "bytes": 1129, "sha256": "1a22dcf9240b795c8f77eb3691b1d4b3d792750d1781e97efb18a64c450b148f" }, { "path": "judge-context-invoice.json", "bytes": 1809, "sha256": "7f1cc39e48ea73100cb4a21d8801474f0117112b2b9eda5cb0f20f3160f395c4" } ], "inputBytes": 19424, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.6 }, "systemMemory": { "sampleCount": 31, "peakUsedMb": 30924, "totalMb": 31865 }, "evidenceHashes": { "result.json": "c16276c68d0fd5d897e9d8c8521cc79084474961bd546c5274cbde0b0ce0cc4a", "score.json": "9100a374ba4ed5c4c1e171cfee9323b7769eb0afef44eaae9d8b33321efac033", "metrics.json": "46e5e1039305bada0f87b8279d5c40bf2e80a1ac2f85412f9e1adcca1a8c1951", "codex-inline-judge.json": "34e0716ef26f460cff9f5162c3c46b2657a3ddd6867a44f9398f3010f713f06c" }, "rubricId": "craftbook-invoice-run", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1", "scenarioId": "craftbook-invoice-run", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 27612 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-invoice-run", "bytes": 555, "score": 10, "scoreMax": 10, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.059838 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 26 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 27612 }, { "source": "facts", "path": "outcome.durationMs", "value": 179514 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-invoice-run", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo", "runDir": "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo", "finishedAt": "2026-10-28T17:04:34.827Z", "rawSuccess": true, "rawReason": "craftbook-invoice-run passed 10 deterministic craftbook checks", "durationMs": 197136, "rawFixedComposite": 9, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 5.4, "judgeArtifactCandidates": [ "workspace/fieldnote-studio-office/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 137.89, "nativePrefillTokensPerSec": 1869.46, "inputTokens": 80801, "outputTokens": 10608, "peakRssMb": 21385, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 197136, "derived": { "durationMs": 183868, "meanTokensPerSec": 137.89, "genTokensPerSec": 137.89, "genTokens": 10608, "promptTokensPerSec": 1869.46, "peakRssMb": 21385, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 66074, "output": 474 } } }, "usage": { "available": true, "totalInputTokens": 80801, "totalOutputTokens": 10608, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 66074, "todayTokensCached": 60882, "todayTokensOut": 474, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 66074, "totalTokensCached": 60882, "totalTokensOut": 474, "totalCost": 0, "medianOutputTokensPerSec": 128.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 128.4, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9729, "cachedInputTokens": 9236, "outputTokens": 75, "cost": 0, "durationMs": 31897, "outputTokensPerSec": 115.33360244507237, "at": "2026-10-28T17:04:18.949Z" }, "lastUpdated": "2026-10-28T17:04:18.949Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:04:24.829Z" } }, "source": "engine-log" }, "process": { "sampleCount": 34, "peakRssMb": 21385, "peakCpuPercent": 267.4, "firstAtMs": 1, "lastAtMs": 182769 }, "gpu": { "available": true, "sampleCount": 34, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 30673, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:01:17.691Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T17:13:29.594Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Independently read this trial's complete report, three invoice HTML sources, clients/ledger and all scope/billables/fanout/verdict artifacts plus captured mutation inventory. Applied exact frozen axes. Ledger amounts and unbroken invoice identities are correct, but books remain at 041/uninvoiced and client terms are not demonstrated by the unexplained due dates. Source-only print assessment, without claiming browser rendering. All selected input bytes are hashed and untruncated under 20 KiB.", "suiteId": "complex-work", "scenarioId": "craftbook-invoice-run", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo", "artifactPath": "workspace/fieldnote-studio-office/report.md", "contextArtifactPaths": [ "workspace/fieldnote-studio-office/clients.json", "workspace/fieldnote-studio-office/ledger.json", "workspace/fieldnote-studio-office/invoices/2026-042.html", "workspace/fieldnote-studio-office/invoices/2026-043.html", "workspace/fieldnote-studio-office/invoices/2026-044.html", "artifacts/fieldnote-studio-office/tasks/1/scope.md", "artifacts/fieldnote-studio-office/tasks/1/billables.json", "artifacts/fieldnote-studio-office/tasks/1/draft.md", "artifacts/fieldnote-studio-office/tasks/1/verdict.md", "judge-context-invoice.json" ], "artifactKind": "markdown", "scoreAxes": { "numeric integrity": 7, "numbering continuity": 8, "professionalism": 2, "completeness": 4, "print-polish": 6 }, "justification": "Correct amounts and sequential numbers cover all three billable clients, with the idle client skipped. Ledger and last number are not updated. Unexplained due dates, contact placeholders, unsupported zero tax and a Bluestem invoice lacking issuer identity reduce readiness. Print rules exist, but the report claims invoices were sent and its verdict supplies no supporting analysis.", "artifactSha256": "b0ff0e73b7f1a3004e33aa284dcf11c62707e07d9ecb02f7252cf52da2f3508a", "artifactBytes": 503, "contextArtifacts": [ { "path": "workspace/fieldnote-studio-office/clients.json", "bytes": 579, "sha256": "1aa5d375dd5ecaaf2470f8a402b6b57c8a1c53508f0b49dccaf9baebf8d63170" }, { "path": "workspace/fieldnote-studio-office/ledger.json", "bytes": 878, "sha256": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-042.html", "bytes": 4894, "sha256": "196a8aa6886980f7753454d4c27f6355925ed4641c6d6067305c43de3f27d063" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-043.html", "bytes": 5276, "sha256": "13724d1fb807d3b1b6bac439dd46b89debccc4b41567a14fac94120b1242fa73" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-044.html", "bytes": 3398, "sha256": "c017136dc568c187533a5c285545247b99140aa19e638b7af74220cf777d64c0" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/scope.md", "bytes": 344, "sha256": "6d99866fbd63cb481338d12d7dd7a6868051fa4e7aa442929fdc17efe1ffc9b3" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/billables.json", "bytes": 538, "sha256": "125dccb2a75bd46a3f9bb54ebff1ac2cfbe5f5021326ad332fc00474915f5a2b" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/draft.md", "bytes": 248, "sha256": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/verdict.md", "bytes": 222, "sha256": "b8bd5342e47768781831bc21c4ffd35e2422487ff85529e2f78f3aa38c36b09a" }, { "path": "judge-context-invoice.json", "bytes": 1665, "sha256": "0ee9d81eee6735f49158346317540c55ff1d42559de4fb8e374303b4b2496ca5" } ], "inputBytes": 18545, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.4 }, "systemMemory": { "sampleCount": 34, "peakUsedMb": 30673, "totalMb": 31865 }, "evidenceHashes": { "result.json": "9283e29416203293afcc5bd430e643488e0872220515f893c2566f1acbfab9d6", "score.json": "3ae0969693719dcbfa1ed7cf42c61cb4fcad5844e0d0f1f9e92ef09644a9611c", "metrics.json": "9e86bcba12efa049f4d0b7a037c640e5afb5bae1c774af2a16cf15d1a35e8f74", "codex-inline-judge.json": "a28ac1b3aa02b66cb24a4a4ee9f07daee23965f9c8f583ad4c2b609131f39987" }, "rubricId": "craftbook-invoice-run", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo", "scenarioId": "craftbook-invoice-run", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26693 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-invoice-run", "bytes": 501, "score": 10, "scoreMax": 10, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.065712 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 25 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26693 }, { "source": "facts", "path": "outcome.durationMs", "value": 197136 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-invoice-run", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom", "runDir": "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom", "finishedAt": "2026-10-28T17:07:48.266Z", "rawSuccess": true, "rawReason": "craftbook-invoice-run passed 10 deterministic craftbook checks", "durationMs": 192438, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "judged", "judgmentMean": 5.6, "judgeArtifactCandidates": [ "workspace/fieldnote-studio-office/report.md" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 142.73, "nativePrefillTokensPerSec": 1821.14, "inputTokens": 87531, "outputTokens": 9139, "peakRssMb": 21991, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 192438, "derived": { "durationMs": 178700, "meanTokensPerSec": 142.73, "genTokensPerSec": 142.73, "genTokens": 9139, "promptTokensPerSec": 1821.14, "peakRssMb": 21991, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 73154, "output": 762 } } }, "usage": { "available": true, "totalInputTokens": 87531, "totalOutputTokens": 9139, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 73154, "todayTokensCached": 68157, "todayTokensOut": 762, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 73154, "totalTokensCached": 68157, "totalTokensOut": 762, "totalCost": 0, "medianOutputTokensPerSec": 142.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 142.2, "turns": 6 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 16966, "cachedInputTokens": 16663, "outputTokens": 363, "cost": 0, "durationMs": 41879, "outputTokensPerSec": 146.42206362079114, "at": "2026-10-28T17:07:37.093Z" }, "lastUpdated": "2026-10-28T17:07:37.093Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:07:37.859Z" } }, "source": "engine-log" }, "process": { "sampleCount": 33, "peakRssMb": 21991, "peakCpuPercent": 277.1, "firstAtMs": 2, "lastAtMs": 177633 }, "gpu": { "available": true, "sampleCount": 33, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 31172, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:04:35.828Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T17:13:29.599Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "scenario", "reviewBasis": "Read this trial's complete required report and every invoice source, own seeded clients/ledger, supporting artifacts and captured mutation inventory. Exact five frozen axes, not inherited scores. Correct line amounts/total/number sequence coexist with unchanged accounting state and unexplained dates. Print-polish is based on complete CSS including @page and print rules, with no visual-render claim. Combined selected inputs are untruncated and hashed below 20 KiB.", "suiteId": "complex-work", "scenarioId": "craftbook-invoice-run", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom", "artifactPath": "workspace/fieldnote-studio-office/report.md", "contextArtifactPaths": [ "workspace/fieldnote-studio-office/clients.json", "workspace/fieldnote-studio-office/ledger.json", "workspace/fieldnote-studio-office/invoices/2026-042.html", "workspace/fieldnote-studio-office/invoices/2026-043.html", "workspace/fieldnote-studio-office/invoices/2026-044.html", "artifacts/fieldnote-studio-office/tasks/1/scope.md", "artifacts/fieldnote-studio-office/tasks/1/billables.json", "artifacts/fieldnote-studio-office/tasks/1/draft.md", "artifacts/fieldnote-studio-office/tasks/1/verdict.md", "judge-context-invoice.json" ], "artifactKind": "markdown", "scoreAxes": { "numeric integrity": 7, "numbering continuity": 8, "professionalism": 3, "completeness": 4, "print-polish": 6 }, "justification": "Three invoices correctly total 5415 with numbers 042-044 and a deliberate idle-client skip. The ledger still says uninvoiced/041. No issue date or terms calculation supports the due dates; known contacts are omitted or placeholders, and zero tax is assumed. Print CSS is present, but the verdict validates dates against model-authored billables rather than source terms.", "artifactSha256": "26e309608ab8421941c90db3e41173e3d78b265971d5a46a68c1aaded60dc68b", "artifactBytes": 491, "contextArtifacts": [ { "path": "workspace/fieldnote-studio-office/clients.json", "bytes": 579, "sha256": "1aa5d375dd5ecaaf2470f8a402b6b57c8a1c53508f0b49dccaf9baebf8d63170" }, { "path": "workspace/fieldnote-studio-office/ledger.json", "bytes": 878, "sha256": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-042.html", "bytes": 4406, "sha256": "65e693b1b537fa5c4560c9c09d27290f03d3a79c7ec6d24039292fcf744f4047" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-043.html", "bytes": 4656, "sha256": "04e66cae823071016b237491c06fe579805031aba24a1339b7d8472c39b0a3ce" }, { "path": "workspace/fieldnote-studio-office/invoices/2026-044.html", "bytes": 4370, "sha256": "7834b0bb6b02475adb057fa3760376ca68e6da2d03d04849a27f5bd02de68078" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/scope.md", "bytes": 339, "sha256": "ca89f0d717bd93d22cc55c0ece845f58b606239724783c4c867e782e6a7d45e1" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/billables.json", "bytes": 538, "sha256": "125dccb2a75bd46a3f9bb54ebff1ac2cfbe5f5021326ad332fc00474915f5a2b" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/draft.md", "bytes": 248, "sha256": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205" }, { "path": "artifacts/fieldnote-studio-office/tasks/1/verdict.md", "bytes": 903, "sha256": "58b1d02de5f60a98b9f7bceb26990cf9c604cb7a8a16df0dd367bfde712656b7" }, { "path": "judge-context-invoice.json", "bytes": 1784, "sha256": "bcd8a0a6b271aa620b616be0aaca4a44128292ad1699c361ec3d5d1b0b2d3df2" } ], "inputBytes": 19192, "inputLimitBytes": 20480, "truncated": false, "meanScore": 5.6 }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 31172, "totalMb": 31865 }, "evidenceHashes": { "result.json": "e0567a9058c1b2ec98e0dbc2e293077b018240df378049144e21e95316b20e41", "score.json": "85a32b2938dff4e8cf1b32e1d42e390d555bf36c669196e926a2f764cde94d67", "metrics.json": "80ad2e9ee3aa2f6994f30943fcb1efce57b4c21a6c63e51359a1f5cce5e15e9d", "codex-inline-judge.json": "3a23eceeeb30648a102f212b7788c0d7b544532d29809cc02cee0364a765d21b" }, "rubricId": "craftbook-invoice-run", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom", "scenarioId": "craftbook-invoice-run", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26894 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-invoice-run", "bytes": 489, "score": 10, "scoreMax": 10, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.064146 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 34 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 26894 }, { "source": "facts", "path": "outcome.durationMs", "value": 192438 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-linear", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb", "finishedAt": "2026-10-28T17:11:15.749Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: CSV Order Cleanup; missing signals stayed unchanged: task order-intake-cleanup/2 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count., out/report.md is 0 bytes, need ≥ 80, out/orders.json does not exist yet. Last failure: task order-intake-cleanup/2 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count.", "durationMs": 203896, "rawFixedComposite": 4.6, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 104.24, "nativePrefillTokensPerSec": 1646.57, "inputTokens": 67102, "outputTokens": 10074, "peakRssMb": 19025, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 203896, "derived": { "durationMs": 194952, "meanTokensPerSec": 104.24, "genTokensPerSec": 104.24, "genTokens": 10074, "promptTokensPerSec": 1646.57, "peakRssMb": 19025, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 138109, "output": 1817 } } }, "usage": { "available": true, "totalInputTokens": 67102, "totalOutputTokens": 10074, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 138109, "todayTokensCached": 127919, "todayTokensOut": 1817, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 138109, "totalTokensCached": 127919, "totalTokensOut": 1817, "totalCost": 0, "medianOutputTokensPerSec": 114.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 114.5, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7929, "cachedInputTokens": 7766, "outputTokens": 62, "cost": 0, "durationMs": 2149, "outputTokensPerSec": 142.61133035330204, "at": "2026-10-28T17:11:06.416Z" }, "lastUpdated": "2026-10-28T17:11:06.416Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:11:10.160Z" } }, "source": "engine-log" }, "process": { "sampleCount": 36, "peakRssMb": 19025, "peakCpuPercent": 268.1, "firstAtMs": 1, "lastAtMs": 193937 }, "gpu": { "available": true, "sampleCount": 36, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 36, "peakUsedMb": 28829, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:07:51.853Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T17:13:31.045Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-linear", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 36, "peakUsedMb": 28829, "totalMb": 31865 }, "evidenceHashes": { "result.json": "7215d7571075b176947685f233bd280ed0184cf13e668cb46d49997d56b7bfc7", "score.json": "efa63d89f3c30b7e58b5f3b06ff11db2817d53e1ee2e7a6f0aa84b3a719410e2", "metrics.json": "ae387c116ef5abd9aa34d8e4111001af0b4f7a64c6dcece347b8e5163fba0ec1", "codex-inline-judge.json": "1cf9a1a72b4e9372b5078946e3b73436fe5e4b651cc366d3d4c4262c7ba24888" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb", "scenarioId": "craftbook-author-linear", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: CSV Order Cleanup; missing signals stayed unchanged: task order-intake-cleanup/2 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count., out/report.md is 0 bytes, need ≥ 80, out/orders.json does not exist yet. Last failure: task order-intake-cleanup/2 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 115634 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-linear", "bytes": 317, "score": 4, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/2 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } } ] }, "quality": { "score": 5.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 57.1% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 115634 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-linear", "bytes": 317, "score": 4, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/2 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.04247833333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 32 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 115634 }, { "source": "facts", "path": "outcome.durationMs", "value": 203896 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.6, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-linear", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s", "finishedAt": "2026-10-28T17:17:46.554Z", "rawSuccess": true, "rawReason": "authored a gated 3-step craftbook, invoked it, ran the task to completion, and all three outputs pass their checks", "durationMs": 389416, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 113.38, "nativePrefillTokensPerSec": 1741.56, "inputTokens": 155958, "outputTokens": 21825, "peakRssMb": 19771, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 389416, "derived": { "durationMs": 379431, "meanTokensPerSec": 113.38, "genTokensPerSec": 113.38, "genTokens": 21825, "promptTokensPerSec": 1741.56, "peakRssMb": 19771, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 295330, "output": 2672 } } }, "usage": { "available": true, "totalInputTokens": 155958, "totalOutputTokens": 21825, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 295330, "todayTokensCached": 285111, "todayTokensOut": 2672, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 295330, "totalTokensCached": 285111, "totalTokensOut": 2672, "totalCost": 0, "medianOutputTokensPerSec": 102.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 102.3, "turns": 15 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10994, "cachedInputTokens": 10797, "outputTokens": 125, "cost": 0, "durationMs": 16103, "outputTokensPerSec": 137.31563438162007, "at": "2026-10-28T17:17:37.962Z" }, "lastUpdated": "2026-10-28T17:17:37.962Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:17:39.966Z" } }, "source": "engine-log" }, "process": { "sampleCount": 69, "peakRssMb": 19771, "peakCpuPercent": 266.7, "firstAtMs": 1, "lastAtMs": 378422 }, "gpu": { "available": true, "sampleCount": 69, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 69, "peakUsedMb": 29382, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:11:17.138Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T17:20:42.279Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-linear", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 69, "peakUsedMb": 29382, "totalMb": 31865 }, "evidenceHashes": { "result.json": "3a346068df0a008d32e17d639c95fb2cd93a61a3a7e41a2168a98acf50da246c", "score.json": "89bd545d14d56ba16e963100bb4d7c2ce2ee56f72baf32bb112f95c6d8d7d34a", "metrics.json": "efe21ddf0625fbf74cd62b627cb10651bc39b5854118bb9cd86cbe4b32e1ddba", "codex-inline-judge.json": "c8b5f2152e587fd5abd3790f9c35083812b8a99655618ee44406d90c47a54277" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s", "scenarioId": "craftbook-author-linear", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 184562 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-linear", "bytes": 5523, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08112833333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 45 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 184562 }, { "source": "facts", "path": "outcome.durationMs", "value": 389416 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-linear", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y", "finishedAt": "2026-10-28T17:35:07.588Z", "rawSuccess": false, "rawReason": "chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 004b00e149342d8c) (re-engage nudge was sent and ignored)", "durationMs": 1039637, "rawFixedComposite": 5.4, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 112.19, "nativePrefillTokensPerSec": 1673.3, "inputTokens": 104395, "outputTokens": 20438, "peakRssMb": 20332, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 1039637, "derived": { "durationMs": 1035395, "meanTokensPerSec": 112.19, "genTokensPerSec": 112.19, "genTokens": 20438, "promptTokensPerSec": 1673.3, "peakRssMb": 20332, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 265139, "output": 7392 } } }, "usage": { "available": true, "totalInputTokens": 104395, "totalOutputTokens": 20438, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 265139, "todayTokensCached": 232710, "todayTokensOut": 7392, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 265139, "totalTokensCached": 232710, "totalTokensOut": 7392, "totalCost": 0, "medianOutputTokensPerSec": 100.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 100.9, "turns": 15 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10596, "cachedInputTokens": 10399, "outputTokens": 93, "cost": 0, "durationMs": 29531, "outputTokensPerSec": 82.45316077918237, "at": "2026-10-28T17:29:59.788Z" }, "lastUpdated": "2026-10-28T17:29:59.788Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:35:06.829Z" } }, "source": "engine-log" }, "process": { "sampleCount": 188, "peakRssMb": 20332, "peakCpuPercent": 290.6, "firstAtMs": 0, "lastAtMs": 1034370 }, "gpu": { "available": true, "sampleCount": 188, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 188, "peakUsedMb": 31142, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "chat-stalled", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:17:47.951Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T17:39:16.062Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-linear", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 188, "peakUsedMb": 31142, "totalMb": 31865 }, "evidenceHashes": { "result.json": "279048432fe402de35b437756e25a778d4f4604d53529cb19ac6a6fb8d331231", "score.json": "02d8ad43ceff2fd7bd786212c653ea3f46be919bd9df108026e64b2e4c6a25e7", "metrics.json": "fbef90f46e8222b3d3ef6314ee434c1399091fb47586e00a9e090224021b0408", "codex-inline-judge.json": "307111b49f3877291a7ff5aade176548e15487e3bfd27dbef80594611daa4e59" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y", "scenarioId": "craftbook-author-linear", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "chat-stalled" }, { "source": "facts", "path": "outcome.reason", "value": "chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 004b00e149342d8c) (re-engage nudge was sent and ignored)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 100388 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-linear", "bytes": 3315, "score": 6, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } } ] }, "quality": { "score": 8.6, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 85.7% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 100388 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-linear", "bytes": 3315, "score": 6, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.21659104166666668 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 43 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 100388 }, { "source": "facts", "path": "outcome.durationMs", "value": 1039637 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.4, "band": "needs-tuning" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-route-multi", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj", "runDir": "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj", "finishedAt": "2026-10-28T17:40:55.017Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/press-release.md; missing signals stayed unchanged: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project. Last failure: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project", "durationMs": 343338, "rawFixedComposite": 5.5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 109.25, "nativePrefillTokensPerSec": 1775.65, "inputTokens": 127998, "outputTokens": 17345, "peakRssMb": 20765, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 343338, "derived": { "durationMs": 335989, "meanTokensPerSec": 109.25, "genTokensPerSec": 109.25, "genTokens": 17345, "promptTokensPerSec": 1775.65, "peakRssMb": 20765, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 197366, "output": 7666 } } }, "usage": { "available": true, "totalInputTokens": 127998, "totalOutputTokens": 17345, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 18, "todayTokensIn": 197366, "todayTokensCached": 158754, "todayTokensOut": 7666, "todayCost": 0, "totalTurns": 18, "totalTokensIn": 197366, "totalTokensCached": 158754, "totalTokensOut": 7666, "totalCost": 0, "medianOutputTokensPerSec": 107.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 107.9, "turns": 18 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 14728, "cachedInputTokens": 14651, "outputTokens": 281, "cost": 0, "durationMs": 6557, "outputTokensPerSec": 166.22685810459825, "at": "2026-10-28T17:40:46.076Z" }, "lastUpdated": "2026-10-28T17:40:46.076Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:40:50.963Z" } }, "source": "engine-log" }, "process": { "sampleCount": 61, "peakRssMb": 20765, "peakCpuPercent": 299.8, "firstAtMs": 0, "lastAtMs": 334938 }, "gpu": { "available": true, "sampleCount": 61, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 61, "peakUsedMb": 30906, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:35:11.679Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T17:51:17.158Z", "suiteId": "complex-work", "scenarioId": "craftbook-route-multi", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 61, "peakUsedMb": 30906, "totalMb": 31865 }, "evidenceHashes": { "result.json": "52f2f9ce4bd323909c29cd818dc740c887c6801bef69b7a49113b853e76de4ed", "score.json": "a5db8b3771993acd0751e869b601bc9739a982ecb71c3e0becec8b728f7569c9", "metrics.json": "ac394f1e55b84e8c1210810b1effb261ba380dd75010c173a5f168eb6d21e468", "codex-inline-judge.json": "38e3711f2341b2d3a4da27288db5c65525c2d67b73ba1e3026072aa70357f93b" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj", "scenarioId": "craftbook-route-multi", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/press-release.md; missing signals stayed unchanged: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project. Last failure: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 79182 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-route-multi", "bytes": 1066, "score": 10, "scoreMax": 11, "signals": [], "failReason": "no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" } } ] }, "quality": { "score": 9.1, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 90.9% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 79182 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-route-multi", "bytes": 1066, "score": 10, "scoreMax": 11, "signals": [], "failReason": "no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07152875 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 38 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 79182 }, { "source": "facts", "path": "outcome.durationMs", "value": 343338 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.5, "band": "needs-tuning" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-route-multi", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl", "runDir": "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl", "finishedAt": "2026-10-28T17:51:25.654Z", "rawSuccess": true, "rawReason": "invoked the bundled press-release recipe, ran the task to completion, and the release carries a dateline, the attributed approved quote, the boilerplate, a contact, and the approved facts rather than the stale draft", "durationMs": 629686, "rawFixedComposite": 8.5, "fixedReportPresent": true, "validationStatus": "original-gate-passed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 114.39, "nativePrefillTokensPerSec": 1790.94, "inputTokens": 175250, "outputTokens": 19254, "peakRssMb": 20721, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 629686, "derived": { "durationMs": 622464, "meanTokensPerSec": 114.39, "genTokensPerSec": 114.39, "genTokens": 19254, "promptTokensPerSec": 1790.94, "peakRssMb": 20721, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 251921, "output": 5765 } } }, "usage": { "available": true, "totalInputTokens": 175250, "totalOutputTokens": 19254, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 17, "todayTokensIn": 251921, "todayTokensCached": 224819, "todayTokensOut": 5765, "todayCost": 0, "totalTurns": 17, "totalTokensIn": 251921, "totalTokensCached": 224819, "totalTokensOut": 5765, "totalCost": 0, "medianOutputTokensPerSec": 104.9, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 104.9, "turns": 17 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 7730, "cachedInputTokens": 7559, "outputTokens": 33, "cost": 0, "durationMs": 11956, "outputTokensPerSec": 150.67119308042544, "at": "2026-10-28T17:51:17.847Z" }, "lastUpdated": "2026-10-28T17:51:17.847Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:51:21.666Z" } }, "source": "engine-log" }, "process": { "sampleCount": 113, "peakRssMb": 20721, "peakCpuPercent": 269.6, "firstAtMs": 3, "lastAtMs": 621671 }, "gpu": { "available": true, "sampleCount": 113, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 113, "peakUsedMb": 31283, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": null, "failureClass": null, "failureClassRule": null, "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:40:55.968Z", "rawEligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T18:00:14.303Z", "suiteId": "complex-work", "scenarioId": "craftbook-route-multi", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 113, "peakUsedMb": 31283, "totalMb": 31865 }, "evidenceHashes": { "result.json": "3238e4f7304f01defe83709b5e3300cdf24d27e378280f48bec174d2563aab53", "score.json": "d713e45e7ec677622a7534fc401a405b4e64e3bbd4642bdd08e4930eb00d77f2", "metrics.json": "b756431bdbf6b506f9e05065e3986c52982af29c5b06cfe4b54b3d8527ba8001", "codex-inline-judge.json": "71b3580e50aa06d576c3923386ab7529f306772dfee0edce7eebba63700c279a" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl", "scenarioId": "craftbook-route-multi", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 433658 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-route-multi", "bytes": 1104, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.13118458333333333 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 53 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 433658 }, { "source": "facts", "path": "outcome.durationMs", "value": 629686 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-route-multi", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5", "runDir": "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5", "finishedAt": "2026-10-28T17:56:30.197Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/press-release.md; missing signals stayed unchanged: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project. Last failure: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project", "durationMs": 303187, "rawFixedComposite": 5.5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 99.49, "nativePrefillTokensPerSec": 1797.46, "inputTokens": 157867, "outputTokens": 13431, "peakRssMb": 20065, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 303187, "derived": { "durationMs": 296969, "meanTokensPerSec": 99.49, "genTokensPerSec": 99.49, "genTokens": 13431, "promptTokensPerSec": 1797.46, "peakRssMb": 20065, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 138037, "output": 3665 } } }, "usage": { "available": true, "totalInputTokens": 157867, "totalOutputTokens": 13431, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 138037, "todayTokensCached": 60299, "todayTokensOut": 3665, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 138037, "totalTokensCached": 60299, "totalTokensOut": 3665, "totalCost": 0, "medianOutputTokensPerSec": 101.7, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 101.7, "turns": 8 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 25919, "outputTokens": 2327, "cost": 0, "durationMs": 37913, "outputTokensPerSec": 99.84923023798717, "at": "2026-10-28T17:56:25.945Z" }, "lastUpdated": "2026-10-28T17:56:25.945Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T17:56:27.370Z" } }, "source": "engine-log" }, "process": { "sampleCount": 54, "peakRssMb": 20065, "peakCpuPercent": 276.7, "firstAtMs": 1, "lastAtMs": 295951 }, "gpu": { "available": true, "sampleCount": 54, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 54, "peakUsedMb": 31261, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:51:27.010Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T18:00:14.317Z", "suiteId": "complex-work", "scenarioId": "craftbook-route-multi", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 54, "peakUsedMb": 31261, "totalMb": 31865 }, "evidenceHashes": { "result.json": "84e04fcfa4bf74e44f5c02ef60879cce65e75648cd3864fbf2d105c61113eef0", "score.json": "98e20965322b59e91967fc42b5119d3e9bf26ea5405b5eeec11df7469993ca30", "metrics.json": "90e4eecc1777fefae7665f59cf9dc47f87cefc932f01e96698e3ffd96fb1d066", "codex-inline-judge.json": "40504f2ed57258aa777d0b549cf98e8602f1fee25832ceefb25f8336620a193f" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5", "scenarioId": "craftbook-route-multi", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/press-release.md; missing signals stayed unchanged: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project. Last failure: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 112683 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-route-multi", "bytes": 1113, "score": 10, "scoreMax": 11, "signals": [], "failReason": "no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" } } ] }, "quality": { "score": 9.1, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 90.9% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 112683 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-route-multi", "bytes": 1113, "score": 10, "scoreMax": 11, "signals": [], "failReason": "no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06316395833333334 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 34 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 112683 }, { "source": "facts", "path": "outcome.durationMs", "value": 303187 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 5.5, "band": "needs-tuning" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-export-generalize", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi", "runDir": "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi", "finishedAt": "2026-10-28T18:09:15.981Z", "rawSuccess": false, "rawReason": "chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 2da6f0fd3a4f605a) (re-engage nudge was sent and ignored)", "durationMs": 761753, "rawFixedComposite": 3.4, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 104.51, "nativePrefillTokensPerSec": 1416.32, "inputTokens": 20916, "outputTokens": 11659, "peakRssMb": 18635, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 761753, "derived": { "durationMs": 757772, "meanTokensPerSec": 104.51, "genTokensPerSec": 104.51, "genTokens": 11659, "promptTokensPerSec": 1416.32, "peakRssMb": 18635, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 78724, "output": 2436 } } }, "usage": { "available": true, "totalInputTokens": 20916, "totalOutputTokens": 11659, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 78724, "todayTokensCached": 77450, "todayTokensOut": 2436, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 78724, "totalTokensCached": 77450, "totalTokensOut": 2436, "totalCost": 0, "medianOutputTokensPerSec": 94.8, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 94.8, "turns": 4 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 20835, "cachedInputTokens": 20613, "outputTokens": 1744, "cost": 0, "durationMs": 19560, "outputTokensPerSec": 92.63079263313631, "at": "2026-10-28T18:04:10.330Z" }, "lastUpdated": "2026-10-28T18:04:10.330Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:09:15.377Z" } }, "source": "engine-log" }, "process": { "sampleCount": 138, "peakRssMb": 18635, "peakCpuPercent": 272, "firstAtMs": 2, "lastAtMs": 756743 }, "gpu": { "available": true, "sampleCount": 138, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 138, "peakUsedMb": 29171, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "chat-stalled", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T17:56:34.228Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T18:10:44.643Z", "suiteId": "complex-work", "scenarioId": "craftbook-export-generalize", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 138, "peakUsedMb": 29171, "totalMb": 31865 }, "evidenceHashes": { "result.json": "a396e3edfc3dff27ad29a246855341afe5d7c08ab1074f52b1d8169aab58b080", "score.json": "182ba35034c5c083e317a3b6c74aa6024720b759cb7bbae56790d4c5007778f8", "metrics.json": "c0c9574554f18a32d865ccfdd38a9dbed71bd1457d1957e04110d176392e5892", "codex-inline-judge.json": "073c077f1a7eae63a44d7f65eca7beaa492bbe8077ac77718d0a79ac8154593f" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi", "scenarioId": "craftbook-export-generalize", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "chat-stalled" }, { "source": "facts", "path": "outcome.reason", "value": "chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 2da6f0fd3a4f605a) (re-engage nudge was sent and ignored)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 55198 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" } } ] }, "quality": { "score": 1.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 16.7% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 55198 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.15869854166666666 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 18 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 55198 }, { "source": "facts", "path": "outcome.durationMs", "value": 761753 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.4, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-export-generalize", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n", "runDir": "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n", "finishedAt": "2026-10-28T18:15:28.328Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: supplier intake recipe; missing signals stayed unchanged: out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time, out/q1-clean.json does not exist yet — write the cleaned Q1 supplier records there as one list of supplier entries, task supplier-intake/1 (from craftbook \"supplier-intake-recipe\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count., out/q2-intake.md does not exist yet — write the Q2 intake report there by running the saved recipe over data/suppliers-q2.json. Last failure: out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time", "durationMs": 371414, "rawFixedComposite": 2.4, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 102.98, "nativePrefillTokensPerSec": 1743.35, "inputTokens": 144685, "outputTokens": 14944, "peakRssMb": 20187, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 371414, "derived": { "durationMs": 359535, "meanTokensPerSec": 102.98, "genTokensPerSec": 102.98, "genTokens": 14944, "promptTokensPerSec": 1743.35, "peakRssMb": 20187, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 135365, "output": 1249 } } }, "usage": { "available": true, "totalInputTokens": 144685, "totalOutputTokens": 14944, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 135365, "todayTokensCached": 112717, "todayTokensOut": 1249, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 135365, "totalTokensCached": 112717, "totalTokensOut": 1249, "totalCost": 0, "medianOutputTokensPerSec": 81.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 81.3, "turns": 7 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 22956, "cachedInputTokens": 22730, "outputTokens": 94, "cost": 0, "durationMs": 15227, "outputTokensPerSec": 81.33757454851086, "at": "2026-10-28T18:15:16.514Z" }, "lastUpdated": "2026-10-28T18:15:16.514Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:15:20.476Z" } }, "source": "engine-log" }, "process": { "sampleCount": 65, "peakRssMb": 20187, "peakCpuPercent": 267.7, "firstAtMs": 0, "lastAtMs": 358429 }, "gpu": { "available": true, "sampleCount": 65, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 65, "peakUsedMb": 30884, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T18:09:16.914Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T18:19:47.075Z", "suiteId": "complex-work", "scenarioId": "craftbook-export-generalize", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 65, "peakUsedMb": 30884, "totalMb": 31865 }, "evidenceHashes": { "result.json": "72f2c074e36b758f792f1f04e2cb8a891f682cf52d8cd40538a88fd4c09a30d2", "score.json": "bf146e2e6532bff65650e5f3c28b8f5a60e4d869bd38e705e3a48444f6aa1f1e", "metrics.json": "1e1848f80c290ce3251c287a7d64841881ede764a120d6aaf4215905f1718fb5", "codex-inline-judge.json": "3c6e2fd16683cfdca114ada4be6ccd441b5dee0ef77f9972b291187fb6214fde" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n", "scenarioId": "craftbook-export-generalize", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: supplier intake recipe; missing signals stayed unchanged: out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time, out/q1-clean.json does not exist yet — write the cleaned Q1 supplier records there as one list of supplier entries, task supplier-intake/1 (from craftbook \"supplier-intake-recipe\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count., out/q2-intake.md does not exist yet — write the Q2 intake report there by running the saved recipe over data/suppliers-q2.json. Last failure: out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 59096 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 73, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" } } ] }, "quality": { "score": 1.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 16.7% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 59096 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 73, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07737791666666667 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 34 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 59096 }, { "source": "facts", "path": "outcome.durationMs", "value": 371414 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 2.4, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-export-generalize", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr", "runDir": "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr", "finishedAt": "2026-10-28T18:29:07.081Z", "rawSuccess": false, "rawReason": "chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 2d44e61df691a47b) (re-engage nudge was sent and ignored)", "durationMs": 817660, "rawFixedComposite": 3.3, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 103.51, "nativePrefillTokensPerSec": 1555.4, "inputTokens": 54410, "outputTokens": 18446, "peakRssMb": 18999, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 817660, "derived": { "durationMs": 813791, "meanTokensPerSec": 103.51, "genTokensPerSec": 103.51, "genTokens": 18446, "promptTokensPerSec": 1555.4, "peakRssMb": 18999, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 231153, "output": 2847 } } }, "usage": { "available": true, "totalInputTokens": 54410, "totalOutputTokens": 18446, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 231153, "todayTokensCached": 220144, "todayTokensOut": 2847, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 231153, "totalTokensCached": 220144, "totalTokensOut": 2847, "totalCost": 0, "medianOutputTokensPerSec": 95.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 95.1, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 24301, "cachedInputTokens": 23940, "outputTokens": 69, "cost": 0, "durationMs": 29839, "outputTokensPerSec": 92.27733682177853, "at": "2026-10-28T18:23:57.470Z" }, "lastUpdated": "2026-10-28T18:23:57.470Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:29:06.432Z" } }, "source": "engine-log" }, "process": { "sampleCount": 148, "peakRssMb": 18999, "peakCpuPercent": 273.2, "firstAtMs": 1, "lastAtMs": 812753 }, "gpu": { "available": true, "sampleCount": 148, "peakUtilPercent": 99, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 148, "peakUsedMb": 29821, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "chat-stalled", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T18:15:29.421Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T18:31:20.840Z", "suiteId": "complex-work", "scenarioId": "craftbook-export-generalize", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 148, "peakUsedMb": 29821, "totalMb": 31865 }, "evidenceHashes": { "result.json": "16326255fa073d52ebc1a200798a19fdfb653bced7fbe7b92e57dfe6f50f7588", "score.json": "43d615493e2f6a86e96246b7cb8eda9db82babebab6156d25c5f9f79b4e090c6", "metrics.json": "08a1c9008ef12bec8ccd768d9e34b7382017efd9c321aa04dd6f66dc38f4e380", "codex-inline-judge.json": "e5469da662ffa054702f9e35ad0c05fddfecf98c11bf3b065d741a14bbe93db4" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr", "scenarioId": "craftbook-export-generalize", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "chat-stalled" }, { "source": "facts", "path": "outcome.reason", "value": "chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 2d44e61df691a47b) (re-engage nudge was sent and ignored)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51588 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 478, "score": 2, "scoreMax": 6, "signals": [], "failReason": "out/q1-clean.json does not exist yet — write the cleaned Q1 supplier records there as one list of supplier entries" } } ] }, "quality": { "score": 3.3, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 33.3% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51588 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 478, "score": 2, "scoreMax": 6, "signals": [], "failReason": "out/q1-clean.json does not exist yet — write the cleaned Q1 supplier records there as one list of supplier entries" } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.17034583333333334 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 25 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 51588 }, { "source": "facts", "path": "outcome.durationMs", "value": 817660 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 3.3, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-params", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7", "finishedAt": "2026-10-28T18:36:18.632Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: out/north-summary.md is 0 bytes, need ≥ 80, out/south-summary.md is 0 bytes, need ≥ 80. Last failure: out/north-summary.md is 0 bytes, need ≥ 80", "durationMs": 427137, "rawFixedComposite": 5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 117.38, "nativePrefillTokensPerSec": 1645, "inputTokens": 120960, "outputTokens": 25560, "peakRssMb": 20256, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 427137, "derived": { "durationMs": 420652, "meanTokensPerSec": 117.38, "genTokensPerSec": 117.38, "genTokens": 25560, "promptTokensPerSec": 1645, "peakRssMb": 20256, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 215875, "output": 2614 } } }, "usage": { "available": true, "totalInputTokens": 120960, "totalOutputTokens": 25560, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 14, "todayTokensIn": 215875, "todayTokensCached": 189150, "todayTokensOut": 2614, "todayCost": 0, "totalTurns": 14, "totalTokensIn": 215875, "totalTokensCached": 189150, "totalTokensOut": 2614, "totalCost": 0, "medianOutputTokensPerSec": 106.3, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 106.3, "turns": 14 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 8540, "outputTokens": 71, "cost": 0, "durationMs": 5435, "outputTokensPerSec": 111.02370197431864, "at": "2026-10-28T18:36:14.497Z" }, "lastUpdated": "2026-10-28T18:36:14.497Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:36:15.504Z" } }, "source": "engine-log" }, "process": { "sampleCount": 76, "peakRssMb": 20256, "peakCpuPercent": 273.3, "firstAtMs": 1, "lastAtMs": 419612 }, "gpu": { "available": true, "sampleCount": 76, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 76, "peakUsedMb": 30856, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T18:29:11.495Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T18:40:46.188Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-params", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 76, "peakUsedMb": 30856, "totalMb": 31865 }, "evidenceHashes": { "result.json": "5bab1cca45e42cd1a45b0663e40b204545de15bde4f8d3bc3296c45abbaf9de4", "score.json": "abb6c55e768de0c080b83164712f2c4b87f430278e83ea0a5f6b882dbdbfdb92", "metrics.json": "32327f51042e2f85318f15d34ebcf6f466f9c6f163ecfe578d9ba70b608f2e0c", "codex-inline-judge.json": "177347cb3592a45b0079288188b2aee29c728e9e44cc5fa5e7fbb3b45a322f22" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7", "scenarioId": "craftbook-author-params", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: out/north-summary.md is 0 bytes, need ≥ 80, out/south-summary.md is 0 bytes, need ≥ 80. Last failure: out/north-summary.md is 0 bytes, need ≥ 80" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 105285 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 65, "score": 5, "scoreMax": 7, "signals": [], "failReason": "out/north-summary.md is 0 bytes, need ≥ 80" } } ] }, "quality": { "score": 7.1, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 71.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 105285 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 65, "score": 5, "scoreMax": 7, "signals": [], "failReason": "out/north-summary.md is 0 bytes, need ≥ 80" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07909944444444444 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 51 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 105285 }, { "source": "facts", "path": "outcome.durationMs", "value": 427137 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5, "band": "needs-tuning" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-params", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr", "finishedAt": "2026-10-28T18:44:16.439Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: no step of craftbook \"regional-sales-summary\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region), out/south-summary.md is 0 bytes, need ≥ 80. Last failure: no step of craftbook \"regional-sales-summary\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region)", "durationMs": 476849, "rawFixedComposite": 5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 110.25, "nativePrefillTokensPerSec": 1753.59, "inputTokens": 185452, "outputTokens": 22230, "peakRssMb": 19269, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 476849, "derived": { "durationMs": 469221, "meanTokensPerSec": 110.25, "genTokensPerSec": 110.25, "genTokens": 22230, "promptTokensPerSec": 1753.59, "peakRssMb": 19269, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 272684, "output": 6055 } } }, "usage": { "available": true, "totalInputTokens": 185452, "totalOutputTokens": 22230, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 18, "todayTokensIn": 272684, "todayTokensCached": 225707, "todayTokensOut": 6055, "todayCost": 0, "totalTurns": 18, "totalTokensIn": 272684, "totalTokensCached": 225707, "totalTokensOut": 6055, "totalCost": 0, "medianOutputTokensPerSec": 100.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 100.2, "turns": 18 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 18743, "outputTokens": 2134, "cost": 0, "durationMs": 39360, "outputTokensPerSec": 118.50154052002677, "at": "2026-10-28T18:44:06.097Z" }, "lastUpdated": "2026-10-28T18:44:06.097Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:44:12.102Z" } }, "source": "engine-log" }, "process": { "sampleCount": 85, "peakRssMb": 19269, "peakCpuPercent": 292.7, "firstAtMs": 0, "lastAtMs": 468023 }, "gpu": { "available": true, "sampleCount": 85, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 85, "peakUsedMb": 30046, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T18:36:19.590Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T18:48:36.285Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-params", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 85, "peakUsedMb": 30046, "totalMb": 31865 }, "evidenceHashes": { "result.json": "2f24a7535d1650de0b549cceac31838f935a32d1c80a6d512999bb4d552d31c7", "score.json": "b4b7f039ae5db9b016abb49a7a63e2317008c660e40e16968ed8d629066f5a9f", "metrics.json": "9da47565854dedc3ce25b1d981f56a72cf0f9e3d7b516e5ac1c34a9b95fa89df", "codex-inline-judge.json": "01cd301567de3efaa28c9c49a1052a0b01bde9b5184ae45579f1ebb648b5f86e" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr", "scenarioId": "craftbook-author-params", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: no step of craftbook \"regional-sales-summary\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region), out/south-summary.md is 0 bytes, need ≥ 80. Last failure: no step of craftbook \"regional-sales-summary\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 193553 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 195, "score": 5, "scoreMax": 7, "signals": [], "failReason": "no step of craftbook \"regional-sales-summary\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region)" } } ] }, "quality": { "score": 7.1, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 71.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 193553 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 195, "score": 5, "scoreMax": 7, "signals": [], "failReason": "no step of craftbook \"regional-sales-summary\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region)" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08830537037037037 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 47 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 193553 }, { "source": "facts", "path": "outcome.durationMs", "value": 476849 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5, "band": "needs-tuning" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-params", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize", "finishedAt": "2026-10-28T18:51:45.500Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: no step of craftbook \"regional-sales-rollup\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region), out/north-summary.md is 0 bytes, need ≥ 80, out/south-summary.md is 0 bytes, need ≥ 80. Last failure: no step of craftbook \"regional-sales-rollup\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region)", "durationMs": 448496, "rawFixedComposite": 4.6, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 114.45, "nativePrefillTokensPerSec": 1700.87, "inputTokens": 152493, "outputTokens": 21065, "peakRssMb": 18649, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 448496, "derived": { "durationMs": 434125, "meanTokensPerSec": 114.45, "genTokensPerSec": 114.45, "genTokens": 21065, "promptTokensPerSec": 1700.87, "peakRssMb": 18649, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 248229, "output": 5147 } } }, "usage": { "available": true, "totalInputTokens": 152493, "totalOutputTokens": 21065, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 15, "todayTokensIn": 248229, "todayTokensCached": 180860, "todayTokensOut": 5147, "todayCost": 0, "totalTurns": 15, "totalTokensIn": 248229, "totalTokensCached": 180860, "totalTokensOut": 5147, "totalCost": 0, "medianOutputTokensPerSec": 102, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 102, "turns": 15 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 15810, "cachedInputTokens": 15387, "outputTokens": 308, "cost": 0, "durationMs": 27838, "outputTokensPerSec": 149.468607454053, "at": "2026-10-28T18:51:33.432Z" }, "lastUpdated": "2026-10-28T18:51:33.432Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T18:51:34.512Z" } }, "source": "engine-log" }, "process": { "sampleCount": 78, "peakRssMb": 18649, "peakCpuPercent": 295.3, "firstAtMs": 1, "lastAtMs": 432998 }, "gpu": { "available": true, "sampleCount": 78, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 78, "peakUsedMb": 30352, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T18:44:17.004Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T18:53:36.534Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-params", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 78, "peakUsedMb": 30352, "totalMb": 31865 }, "evidenceHashes": { "result.json": "4e7a96d3110c3cd802c35fa50878d49a65974defea8c6372c9d1cae3dc0a14c9", "score.json": "e45fa7b5d327c4606d475263c49106c3655945525f25683748b53b15b0011b40", "metrics.json": "5ad4da215c9eb2aad42faec8d644234c201db686cc0c2f610946dfeda62a9600", "codex-inline-judge.json": "6830e4f92ecb0df94c90db36af1b89fb0b34936c06ff2902559d3e32f9580800" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize", "scenarioId": "craftbook-author-params", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: no step of craftbook \"regional-sales-rollup\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region), out/north-summary.md is 0 bytes, need ≥ 80, out/south-summary.md is 0 bytes, need ≥ 80. Last failure: no step of craftbook \"regional-sales-rollup\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 270871 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 63, "score": 4, "scoreMax": 7, "signals": [], "failReason": "no step of craftbook \"regional-sales-rollup\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region)" } } ] }, "quality": { "score": 5.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 57.1% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 270871 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 63, "score": 4, "scoreMax": 7, "signals": [], "failReason": "no step of craftbook \"regional-sales-rollup\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region)" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08305481481481482 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 37 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 270871 }, { "source": "facts", "path": "outcome.durationMs", "value": 448496 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.6, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p", "finishedAt": "2026-10-28T19:02:24.815Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template. Last failure: no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template", "durationMs": 635005, "rawFixedComposite": 5.2, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 97.1, "nativePrefillTokensPerSec": 1443.85, "inputTokens": 142559, "outputTokens": 26288, "peakRssMb": 19488, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 635005, "derived": { "durationMs": 622896, "meanTokensPerSec": 97.1, "genTokensPerSec": 97.1, "genTokens": 26288, "promptTokensPerSec": 1443.85, "peakRssMb": 19488, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 460532, "output": 4165 } } }, "usage": { "available": true, "totalInputTokens": 142559, "totalOutputTokens": 26288, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 18, "todayTokensIn": 460532, "todayTokensCached": 443446, "todayTokensOut": 4165, "todayCost": 0, "totalTurns": 18, "totalTokensIn": 460532, "totalTokensCached": 443446, "totalTokensOut": 4165, "totalCost": 0, "medianOutputTokensPerSec": 79.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 79.4, "turns": 18 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 23510, "cachedInputTokens": 23445, "outputTokens": 414, "cost": 0, "durationMs": 9865, "outputTokensPerSec": 138.44820679401892, "at": "2026-10-28T19:02:15.219Z" }, "lastUpdated": "2026-10-28T19:02:15.219Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T19:02:17.007Z" } }, "source": "engine-log" }, "process": { "sampleCount": 111, "peakRssMb": 19488, "peakCpuPercent": 267.6, "firstAtMs": 3, "lastAtMs": 621770 }, "gpu": { "available": true, "sampleCount": 111, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 111, "peakUsedMb": 31530, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T18:51:49.810Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T19:08:53.699Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 111, "peakUsedMb": 31530, "totalMb": 31865 }, "evidenceHashes": { "result.json": "bc3a7fd516a8ea8f4526e47adc37dd710487c3dfc1af1192afcd60cf0dc49953", "score.json": "b8dc7f30661862be90b44927c4adb68727f4c2f89da548603cbd7da5fd0ea11a", "metrics.json": "a6033e933b929946796af32fef5eeb9c53c8a42db26a33c2eeb8dbfc40181656", "codex-inline-judge.json": "9e6d424c25f7155922a816ce2da43f57db1cfde884d7e8a9e637f673446d5f1d" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p", "scenarioId": "craftbook-author-gate-script", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template. Last failure: no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 404856 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 186, "score": 4, "scoreMax": 5, "signals": [], "failReason": "no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template" } } ] }, "quality": { "score": 8, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 80% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 404856 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 186, "score": 4, "scoreMax": 5, "signals": [], "failReason": "no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08819513888888889 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 46 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 404856 }, { "source": "facts", "path": "outcome.durationMs", "value": 635005 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.2, "band": "needs-tuning" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f", "finishedAt": "2026-10-28T19:07:37.510Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: craftbook \"inventory-health-check\" step \"build\" references gate script \"verifyInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself, task inventory-health-check/1 (from craftbook \"inventory-health-check\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count.. Last failure: craftbook \"inventory-health-check\" step \"build\" references gate script \"verifyInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself", "durationMs": 311243, "rawFixedComposite": 5.2, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 101.74, "nativePrefillTokensPerSec": 1686.19, "inputTokens": 154192, "outputTokens": 13479, "peakRssMb": 19753, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 311243, "derived": { "durationMs": 300650, "meanTokensPerSec": 101.74, "genTokensPerSec": 101.74, "genTokens": 13479, "promptTokensPerSec": 1686.19, "peakRssMb": 19753, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 274359, "output": 3486 } } }, "usage": { "available": true, "totalInputTokens": 154192, "totalOutputTokens": 13479, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 274359, "todayTokensCached": 177944, "todayTokensOut": 3486, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 274359, "totalTokensCached": 177944, "totalTokensOut": 3486, "totalCost": 0, "medianOutputTokensPerSec": 88, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 88, "turns": 13 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9599, "cachedInputTokens": 9225, "outputTokens": 310, "cost": 0, "durationMs": 19681, "outputTokensPerSec": 161.7967926534831, "at": "2026-10-28T19:07:27.075Z" }, "lastUpdated": "2026-10-28T19:07:27.075Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T19:07:31.113Z" } }, "source": "engine-log" }, "process": { "sampleCount": 54, "peakRssMb": 19753, "peakCpuPercent": 297.4, "firstAtMs": 3, "lastAtMs": 299557 }, "gpu": { "available": true, "sampleCount": 54, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 54, "peakUsedMb": 31213, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "infra", "failureClassRule": "gate-unevaluable", "failureClassEvidence": "could not be evaluated — pausing without consuming an attempt: Gate script \"verifyInventoryReport\" could not run: script \"verifyInventoryRep", "nativeEngineIncidents": null, "reviewedEligibility": { "suiteId": "complex-work", "scenarioId": "craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f", "status": "excluded", "category": "authored-gate-unevaluable", "reason": "Retain the original infra/gate-unevaluable exclusion: the model-authored recipe references verifyInventoryReport without carrying runnable script source, so its gate cannot evaluate and the task pauses. This is an authored workflow defect, not evidence of an external machine/setup outage.", "rawFailureClass": "infra", "rawFailureClassRule": "gate-unevaluable", "evidence": [ "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f/artifact-evidence-review.json records hashes and direct source/trace findings.", "craftbook_write at19:03:10 uses scope:standard and code in description;19:04:17 changes library scope but still embeds no script source.", "advance_task_step at19:05:17 and task notes19:05:24 report verifyInventoryReport missing from standard scope; raw score.json eligibility.includedInModelAggregate=false." ], "diagnosis": "Keep the failed attempt in all129 outcome counts and performance, retain raw fixed score5.2 and explicit qualitative N/A. Exclude only the capability aggregate under the existing raw infrastructure rule. Final report must separately count model-authored unevaluable gates so exclusion is not portrayed as an innocent environment failure or successful authoring.", "reviewedAt": "2026-10-28T19:11:20.677Z" }, "startedAt": "2026-10-28T19:02:26.267Z", "rawEligibility": { "failureClass": "infra", "includedInModelAggregate": false, "failureClassRule": "gate-unevaluable", "evidence": [ { "source": "result", "path": "failureClass", "value": "infra" } ] }, "finalCapabilityEligible": false, "finalExclusions": [ { "category": "authored-gate-unevaluable", "reason": "Retain the original infra/gate-unevaluable exclusion: the model-authored recipe references verifyInventoryReport without carrying runnable script source, so its gate cannot evaluate and the task pauses. This is an authored workflow defect, not evidence of an external machine/setup outage." } ], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T19:08:53.711Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 54, "peakUsedMb": 31213, "totalMb": 31865 }, "evidenceHashes": { "result.json": "6653487baa8f538739811eb4e08e4656a0840ab10c09ddfc42d49cfd3cf1e171", "score.json": "efb6d5c7ce5486649ba66d5811cabefa927d297731659d21d991b35717e79039", "metrics.json": "8d83a9049d378fb388680a6dff31673aea507f21dc5fb53ef9c662a2a874e24e", "codex-inline-judge.json": "249e7e947379fc52b48fa32555536ba6dac530fd6fa8cb926734b70440faa7a4" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f", "scenarioId": "craftbook-author-gate-script", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "infra", "includedInModelAggregate": false, "failureClassRule": "gate-unevaluable", "evidence": [ { "source": "result", "path": "failureClass", "value": "infra" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: craftbook \"inventory-health-check\" step \"build\" references gate script \"verifyInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself, task inventory-health-check/1 (from craftbook \"inventory-health-check\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count.. Last failure: craftbook \"inventory-health-check\" step \"build\" references gate script \"verifyInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 161444 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 603, "score": 3, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-check\" step \"build\" references gate script \"verifyInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" } } ] }, "quality": { "score": 6, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 60% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 161444 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 603, "score": 3, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-check\" step \"build\" references gate script \"verifyInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.043228194444444444 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 29 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 161444 }, { "source": "facts", "path": "outcome.durationMs", "value": 311243 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.2, "band": "needs-tuning" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4", "finishedAt": "2026-10-28T19:50:43.105Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: craftbook \"inventory-health-check\" has no step gated by a scope:\"craftbook\" script — attach the custom check script to the build step's gate. Last failure: craftbook \"inventory-health-check\" has no step gated by a scope:\"craftbook\" script — attach the custom check script to the build step's gate", "durationMs": 2585107, "rawFixedComposite": 4.5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 2.25, "judgeArtifactCandidates": [ "workspace/inventory-health-check/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 116.21, "nativePrefillTokensPerSec": 1681.62, "inputTokens": 712129, "outputTokens": 94226, "peakRssMb": 20820, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10521, "gpuMemoryTotalMb": 12227, "durationMs": 2585107, "derived": { "durationMs": 2579667, "meanTokensPerSec": 116.21, "genTokensPerSec": 116.21, "genTokens": 94226, "promptTokensPerSec": 1681.62, "peakRssMb": 20820, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 789428, "output": 25274 } } }, "usage": { "available": true, "totalInputTokens": 712129, "totalOutputTokens": 94226, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 49, "todayTokensIn": 789428, "todayTokensCached": 590009, "todayTokensOut": 25274, "todayCost": 0, "totalTurns": 49, "totalTokensIn": 789428, "totalTokensCached": 590009, "totalTokensOut": 25274, "totalCost": 0, "medianOutputTokensPerSec": 113.1, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 113.1, "turns": 49 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 47248, "cachedInputTokens": 43166, "outputTokens": 1361, "cost": 0, "durationMs": 53312, "outputTokensPerSec": 137.92823674976165, "at": "2026-10-28T19:50:37.173Z" }, "lastUpdated": "2026-10-28T19:50:37.173Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T19:50:41.146Z" } }, "source": "engine-log" }, "process": { "sampleCount": 461, "peakRssMb": 20820, "peakCpuPercent": 278.8, "firstAtMs": 1, "lastAtMs": 2578576 }, "gpu": { "available": true, "sampleCount": 461, "peakUtilPercent": 100, "peakMemUsedMb": 10521, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 461, "peakUsedMb": 31687, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "tool-repeat-abort-storm", "failureClassEvidence": "3× read-tool repeat aborts in one trial", "nativeEngineIncidents": null, "startedAt": "2026-10-28T19:07:37.998Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "tool-repeat-abort-storm", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T20:01:45.314Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Read the complete generated HTML, its final JSON input, seeded stock data, and frozen original authoring prompt. Applied the four exact fallback HTML axes. Static source review only; no browser rendering or runtime execution claimed. The JSON object versus data.forEach mismatch and field-name/status mismatches are directly visible in these inputs. All selected artifact/context bytes are untruncated and hashed under 20 KiB.", "suiteId": "complex-work", "scenarioId": "craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4", "artifactPath": "workspace/inventory-health-check/index.html", "contextArtifactPaths": [ "workspace/inventory-health-check/out/inventory-report.json", "workspace/inventory-health-check/data/inventory.json" ], "artifactKind": "html", "scoreAxes": { "visualQuality": 4, "functionalCompleteness": 1, "codeQuality": 2, "polish": 2 }, "justification": "Cards and a styled table provide a basic layout, but data.forEach expects an array while the saved report is an object. Fields/statuses also disagree, so it cannot display the supplied report. Raw HTML interpolation lacks escaping. Error text exists, but there is no working report view, and this extra page does not provide the requested embedded craftbook check.", "artifactSha256": "17ecc31c979781db38121622cde13f8f69ebb46b256cd5bae872a6c5a8cf5418", "artifactBytes": 7017, "contextArtifacts": [ { "path": "workspace/inventory-health-check/out/inventory-report.json", "bytes": 1895, "sha256": "285eeba48052b57593bd41c0fec980caf1afbb7e647cd4d7bf3314bc1d955084" }, { "path": "workspace/inventory-health-check/data/inventory.json", "bytes": 1006, "sha256": "2a6f44cb0bb8d193c2abf527264e828fbf882e8a744180a786e784fe0688b439" } ], "inputBytes": 9918, "inputLimitBytes": 20480, "truncated": false, "meanScore": 2.25 }, "systemMemory": { "sampleCount": 461, "peakUsedMb": 31687, "totalMb": 31865 }, "evidenceHashes": { "result.json": "5384e48c17010d2211c93aad3e12e43c00dd7e1f260958c5a07abb7e5d3a7c00", "score.json": "26e31da51427f13b9a659d26e3893c69198282ce10d3aad08732275a5a8311c3", "metrics.json": "9e98d1480b0be97e800862e7ed527eec3b300fc2b51167e460c63db37a2aadc9", "codex-inline-judge.json": "5c45ccd2de64d52a15fe3ca311bc013fac58083e4320a6b4f0ad5adbc2e32b14" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4", "scenarioId": "craftbook-author-gate-script", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "tool-repeat-abort-storm", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: craftbook \"inventory-health-check\" has no step gated by a scope:\"craftbook\" script — attach the custom check script to the build step's gate. Last failure: craftbook \"inventory-health-check\" has no step gated by a scope:\"craftbook\" script — attach the custom check script to the build step's gate" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 185654 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 1917, "score": 4, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-check\" has no step gated by a scope:\"craftbook\" script — attach the custom check script to the build step's gate" } } ] }, "quality": { "score": 8, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 80% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 185654 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 1917, "score": 4, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-check\" has no step gated by a scope:\"craftbook\" script — attach the custom check script to the build step's gate" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.3590426388888889 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 177 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 185654 }, { "source": "facts", "path": "outcome.durationMs", "value": 2585107 } ] }, "behavior": { "score": 5, "ruleId": "B_CLEAN_HIGH_INTERVENTION", "summary": "No red flags or missing roles were observed, but the team needed at least three interventions.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 3 } ] } }, "composite": 4.5, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-fanout", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721", "finishedAt": "2026-10-28T20:18:47.096Z", "rawSuccess": false, "rawReason": "chat stalled for 305s — daemon reachable but issuing no model turns (soft digest stuck at 82f1867183be8622) (re-engage nudge was sent and ignored)", "durationMs": 1680133, "rawFixedComposite": 3, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "judged", "judgmentMean": 3, "judgeArtifactCandidates": [ "workspace/store-health-sweep/index.html" ], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 98.25, "nativePrefillTokensPerSec": 1481.49, "inputTokens": 112856, "outputTokens": 36018, "peakRssMb": 19752, "gpuAvailable": true, "peakGpuUtilPercent": 99, "peakGpuMemoryMb": 10521, "gpuMemoryTotalMb": 12227, "durationMs": 1680133, "derived": { "durationMs": 1672145, "meanTokensPerSec": 98.25, "genTokensPerSec": 98.25, "genTokens": 36018, "promptTokensPerSec": 1481.49, "peakRssMb": 19752, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 502602, "output": 5209 } } }, "usage": { "available": true, "totalInputTokens": 112856, "totalOutputTokens": 36018, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 28, "todayTokensIn": 502602, "todayTokensCached": 480413, "todayTokensOut": 5209, "todayCost": 0, "totalTurns": 28, "totalTokensIn": 502602, "totalTokensCached": 480413, "totalTokensOut": 5209, "totalCost": 0, "medianOutputTokensPerSec": 88.4, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 88.4, "turns": 28 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 10393, "cachedInputTokens": 10188, "outputTokens": 364, "cost": 0, "durationMs": 4094, "outputTokensPerSec": 102.43501866687738, "at": "2026-10-28T20:13:39.512Z" }, "lastUpdated": "2026-10-28T20:13:39.512Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T20:18:46.202Z" } }, "source": "engine-log" }, "process": { "sampleCount": 300, "peakRssMb": 19752, "peakCpuPercent": 295.1, "firstAtMs": 0, "lastAtMs": 1671096 }, "gpu": { "available": true, "sampleCount": 300, "peakUtilPercent": 99, "peakMemUsedMb": 10521, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 300, "peakUsedMb": 31192, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "chat-stalled", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T19:50:46.963Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "judgedAt": "2026-10-28T20:22:50.281Z", "modelId": "gemma4-12b-q4", "status": "judged", "rubric": "default-html", "reviewBasis": "Read complete generated HTML, own seeded store list, loose Python generator and both actual report artifacts, plus the frozen user prompt. Applied the exact four fallback HTML axes. Source/CSS review only; no browser execution or visual-render claim. HTML explicitly simulates reports using hard-coded four-store data. The generator is source evidence, not proof of execution. All selected inputs are hashed and untruncated below20KiB.", "suiteId": "complex-work", "scenarioId": "craftbook-author-fanout", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721", "artifactPath": "workspace/store-health-sweep/index.html", "contextArtifactPaths": [ "workspace/store-health-sweep/data/stores.json", "workspace/store-health-sweep/sweep.py", "workspace/store-health-sweep/out/store-1-health.md", "artifacts/store-health-sweep/out/harbor-health.md" ], "artifactKind": "html", "scoreAxes": { "visualQuality": 4, "functionalCompleteness": 2, "codeQuality": 4, "polish": 2 }, "justification": "Simple cards show the four seeded stores and their correct counts, but all data and risk labels are hard-coded. The page neither reads a growing store list nor produces the requested files. A separate Python generator exists without execution evidence. Generic repeated advice and basic styling add little polish; the only written reports use the wrong path or storage scope.", "artifactSha256": "efdddbca93c438ccaa0b9317d280b4ffbc8c8c1a338b89e68cf98e1571e4b9e8", "artifactBytes": 3166, "contextArtifacts": [ { "path": "workspace/store-health-sweep/data/stores.json", "bytes": 517, "sha256": "8bf728bad631198134c68817a9d08eaf08804c45fe27af05c1d8433c7e573e22" }, { "path": "workspace/store-health-sweep/sweep.py", "bytes": 2101, "sha256": "d598e70af566ff3ffd2a9127755818555be869809f49a82e214cdf877d289194" }, { "path": "workspace/store-health-sweep/out/store-1-health.md", "bytes": 451, "sha256": "724d1399625b8769f7a20dc266d34e140b4d7b5c32bde433d31858a6eaec4254" }, { "path": "artifacts/store-health-sweep/out/harbor-health.md", "bytes": 181, "sha256": "8b3d2c81b6c85386a3c222e348350ee00b9ffd3426c0fc18339ec722d61f8236" } ], "inputBytes": 6416, "inputLimitBytes": 20480, "truncated": false, "meanScore": 3 }, "systemMemory": { "sampleCount": 300, "peakUsedMb": 31192, "totalMb": 31865 }, "evidenceHashes": { "result.json": "316ae744d16e0ad2179ce1487cb69d51d75e75c07d29de544a2a6c752250368c", "score.json": "5428e1e65b3ce8aed11755ec6c94da719c7ba4cb324922b01fddade4266aaa1e", "metrics.json": "823002be847c747e7e718ca171a469d774bb5742b8047fc6ece33bcc3db64aa2", "codex-inline-judge.json": "1d9ab3569d2488122eab76a5ad198299debdfcbafd57f5a74d139ba3987473a9" }, "rubricId": "default-html-v1", "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721", "scenarioId": "craftbook-author-fanout", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "chat-stalled" }, { "source": "facts", "path": "outcome.reason", "value": "chat stalled for 305s — daemon reachable but issuing no model turns (soft digest stuck at 82f1867183be8622) (re-engage nudge was sent and ignored)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 484422 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-fanout", "bytes": 46, "score": 3, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" } } ] }, "quality": { "score": 3.8, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 37.5% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 484422 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-fanout", "bytes": 46, "score": 3, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.23335180555555557 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 61 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 484422 }, { "source": "facts", "path": "outcome.durationMs", "value": 1680133 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 3, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-fanout", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c", "finishedAt": "2026-10-28T20:27:31.046Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: store health sweep; missing signals stayed unchanged: craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store, craftbook \"store-health-sweep\" has produced 1 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run, out/northside-health.md is 0 bytes, need ≥ 60, out/milo-health.md is 0 bytes, need ≥ 60. Last failure: craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store", "durationMs": 522897, "rawFixedComposite": 4.5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 109.22, "nativePrefillTokensPerSec": 1674.13, "inputTokens": 83316, "outputTokens": 17210, "peakRssMb": 19026, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 522897, "derived": { "durationMs": 513612, "meanTokensPerSec": 109.22, "genTokensPerSec": 109.22, "genTokens": 17210, "promptTokensPerSec": 1674.13, "peakRssMb": 19026, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 162957, "output": 5917 } } }, "usage": { "available": true, "totalInputTokens": 83316, "totalOutputTokens": 17210, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 162957, "todayTokensCached": 136679, "todayTokensOut": 5917, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 162957, "totalTokensCached": 136679, "totalTokensOut": 5917, "totalCost": 0, "medianOutputTokensPerSec": 110.5, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 110.5, "turns": 12 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 9380, "cachedInputTokens": 9014, "outputTokens": 310, "cost": 0, "durationMs": 4246, "outputTokensPerSec": 153.05493186083592, "at": "2026-10-28T20:27:23.810Z" }, "lastUpdated": "2026-10-28T20:27:23.810Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T20:27:25.474Z" } }, "source": "engine-log" }, "process": { "sampleCount": 93, "peakRssMb": 19026, "peakCpuPercent": 274.5, "firstAtMs": 0, "lastAtMs": 512554 }, "gpu": { "available": true, "sampleCount": 93, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 93, "peakUsedMb": 30585, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T20:18:48.149Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T20:28:41.592Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-fanout", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 93, "peakUsedMb": 30585, "totalMb": 31865 }, "evidenceHashes": { "result.json": "b3b310334889f5c909860236c2c41f4d9d4ade4f92701e8870adbec5f13f6c00", "score.json": "389836d285afc421377473bb2124bcc8f5c257a555cd85bd1fbe7869260950e6", "metrics.json": "127cc296de94249e240fc7340a6b2828615378878adf2f4356d6b174cc3c9657", "codex-inline-judge.json": "ad0ac0b15f2071f0639c34c48af3fd8cf68d88661c79ba3d0582459a1b2703eb" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c", "scenarioId": "craftbook-author-fanout", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: store health sweep; missing signals stayed unchanged: craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store, craftbook \"store-health-sweep\" has produced 1 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run, out/northside-health.md is 0 bytes, need ≥ 60, out/milo-health.md is 0 bytes, need ≥ 60. Last failure: craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 417850 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-fanout", "bytes": 487, "score": 4, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" } } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 417850 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-fanout", "bytes": 487, "score": 4, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07262458333333334 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 30 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 417850 }, { "source": "facts", "path": "outcome.durationMs", "value": 522897 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.5, "band": "framework-gap" } }, { "suiteId": "complex-work", "scenarioId": "craftbook-author-fanout", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556", "runDir": "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556", "finishedAt": "2026-10-28T20:39:34.118Z", "rawSuccess": false, "rawReason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: store health sweep; missing signals stayed unchanged: no reusable authored craftbook exists yet — author the store health sweep as a craftbook template, not as ad-hoc chat work, out/harbor-health.md is 0 bytes, need ≥ 60, out/linden-health.md is 0 bytes, need ≥ 60, out/northside-health.md is 0 bytes, need ≥ 60. Last failure: no reusable authored craftbook exists yet — author the store health sweep as a craftbook template, not as ad-hoc chat work", "durationMs": 721741, "rawFixedComposite": 3.5, "fixedReportPresent": true, "validationStatus": "original-trial-failed", "judgmentStatus": "not-applicable", "judgmentMean": null, "judgeArtifactCandidates": [], "performancePresent": true, "performance": { "tokenSource": "engine-log", "nativeDecodeTokensPerSec": 99.32, "nativePrefillTokensPerSec": 1259.77, "inputTokens": 31391, "outputTokens": 17780, "peakRssMb": 18725, "gpuAvailable": true, "peakGpuUtilPercent": 100, "peakGpuMemoryMb": 10515, "gpuMemoryTotalMb": 12227, "durationMs": 721741, "derived": { "durationMs": 707755, "meanTokensPerSec": 99.32, "genTokensPerSec": 99.32, "genTokens": 17780, "promptTokensPerSec": 1259.77, "peakRssMb": 18725, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 237292, "output": 2728 } } }, "usage": { "available": true, "totalInputTokens": 31391, "totalOutputTokens": 17780, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 237292, "todayTokensCached": 234472, "todayTokensOut": 2728, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 237292, "totalTokensCached": 234472, "totalTokensOut": 2728, "totalCost": 0, "medianOutputTokensPerSec": 77.2, "modelSpeeds": [ { "model": "gemma4-12b-q4", "medianOutputTokensPerSec": 77.2, "turns": 10 } ], "lastTurn": { "model": "gemma4-12b-q4", "inputTokens": 31255, "cachedInputTokens": 31122, "outputTokens": 98, "cost": 0, "durationMs": 9841, "outputTokensPerSec": 72.52846181339099, "at": "2026-10-28T20:39:21.704Z" }, "lastUpdated": "2026-10-28T20:39:21.704Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-10-28T20:39:23.678Z" } }, "source": "engine-log" }, "process": { "sampleCount": 128, "peakRssMb": 18725, "peakCpuPercent": 266.7, "firstAtMs": 1, "lastAtMs": 706514 }, "gpu": { "available": true, "sampleCount": 128, "peakUtilPercent": 100, "peakMemUsedMb": 10515, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 128, "peakUsedMb": 30333, "totalMb": 31865 } }, "engineContext": { "launches": 1, "grantedPerSlotTokens": 157696, "totalTokens": 157696, "slots": 1, "kvCacheType": "f16", "launchModel": "gemma4-12b-q4" }, "failureMode": "model-stuck", "failureClass": "model", "failureClassRule": "model-default", "failureClassEvidence": null, "nativeEngineIncidents": null, "startedAt": "2026-10-28T20:27:32.377Z", "rawEligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "finalCapabilityEligible": true, "finalExclusions": [], "runtime": null, "judge": { "schemaVersion": 1, "judgeProvider": "codex-inline", "judgeModel": "current-session", "assessedAt": "2026-10-28T20:41:24.111Z", "suiteId": "complex-work", "scenarioId": "craftbook-author-fanout", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556", "modelId": "gemma4-12b-q4", "status": "not-applicable", "reason": "Registered scenario has no declared judge spec and this completed trial has no HTML artifact for the legacy HTML rubric. Deterministic gates are scored separately.", "judgePlanSha256": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "scoreAxes": null, "meanScore": null }, "systemMemory": { "sampleCount": 128, "peakUsedMb": 30333, "totalMb": 31865 }, "evidenceHashes": { "result.json": "25d15601798d72efeff6ebda4608f31f6f969181542e8bfeafb701350fe14658", "score.json": "caad74e530ee12f35e33dfeaf85109355f7be6f3317f28d224193ac37a66228e", "metrics.json": "67a81c5e517847ee768690e5df7f1434b5e5fdc1652550e12b688d7daa5d7773", "codex-inline-judge.json": "e40984d8ea7c00873203d1fea3703c331cb5c3dc867becfb32040ae38d5a2439" }, "rubricId": null, "fixedScore": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556", "scenarioId": "craftbook-author-fanout", "modelId": "gemma4-12b-q4", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: store health sweep; missing signals stayed unchanged: no reusable authored craftbook exists yet — author the store health sweep as a craftbook template, not as ad-hoc chat work, out/harbor-health.md is 0 bytes, need ≥ 60, out/linden-health.md is 0 bytes, need ≥ 60, out/northside-health.md is 0 bytes, need ≥ 60. Last failure: no reusable authored craftbook exists yet — author the store health sweep as a craftbook template, not as ad-hoc chat work" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 102901 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-fanout", "bytes": 0, "score": 3, "scoreMax": 8, "signals": [], "failReason": "no reusable authored craftbook exists yet — author the store health sweep as a craftbook template, not as ad-hoc chat work" } } ] }, "quality": { "score": 3.8, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 37.5% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 102901 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-fanout", "bytes": 0, "score": 3, "scoreMax": 8, "signals": [], "failReason": "no reusable authored craftbook exists yet — author the store health sweep as a craftbook template, not as ad-hoc chat work" } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.10024180555555555 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 26 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 102901 }, { "source": "facts", "path": "outcome.durationMs", "value": 721741 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.5, "band": "framework-gap" } } ], "evidence": { "availability": "Raw transcripts and artifacts remain in the local evaluation archive; they are not included in this download.", "pathConvention": "Evidence paths are relative to the archived run. Workstation paths and hostname are redacted.", "sourceFiles": { "final-accounting.json": "f0163178da0cff20aaa89523348cb4b3e230840d8733dffefc7b2aaad0388120", "launch-provenance.json": "2bdaf63b29c4041f6edac56f514efd068f26289d32649aa9b28991fc13f4a192", "judge-plan.json": "769d9734b46ea6cb311820d0e73bf46379b5cf3a367bda944fc5832365f4cd55", "posthoc-runtime/summary.json": "29ac37f3ffcba23dde3a73b76ca6493d26f3f274109c16cfd534fce18054947c", "eligibility-review.json": "88800e771544b6e9a65046718dce6b96cb848e1d4d120c0cc162231271c3d138", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv/result.json": "b000d7f49c200804c09bb26e95f3ce190b25b7f78f8c2c6a33bb7b6a2c53d02f", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv/score.json": "ee4fecf5d49f3027f3a1667f1bab100f0edf24b9fc054f528897db7a71973e6b", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv/metrics.json": "d74b1e9525538300384f0919c3cdb8fcada258545449b4f7def6aa0e850cb932", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv/codex-inline-judge.json": "c6776c3e430de19a7c00e30b1ad43c3e20f1d584d56ccba890849716f19cb671", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv/workspace/this-work-e/index.html": "4c540405bb191275fd03e1b5a769ca2f952bd461fee9538cd4d3680edd93814b", "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-05-12-793Z-ppfv.json": "d7aedc6ef36af356b001571df39ccff66d7a11d35e30278f300f0302f16f9d40", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q/result.json": "86d931bc3a664283d9b14208f232fbf907bfac5171c922fa602378ba18d7ad71", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q/score.json": "017db071966144ec19bf27e79789704ace15f12d23ac3409adbaff50ec32081a", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q/metrics.json": "5b6fd7f663b40f8f98a5deb57fd7672e5f0532cec584556ca9c73a46ee6d4798", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q/codex-inline-judge.json": "e2909baed8dbf0cfd69a90676f135533452c12f0a2edf6a882d67ea16c508184", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q/workspace/tic-tac-toe-game/index.html": "acbfb6db83b99860ef1280e38fdb61c35096f3c796cfa6e3492b2d86d118bed7", "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q.json": "33e986adc2d80177af0d999858370636ecab82ef43ba7aa3f540c93864918749", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/result.json": "cf0d75e4c814050ce1e5ded8414034f121e85c30152f3b07d1bc34aaf9387442", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/score.json": "272cfce4c10cbf81431056e3e90c0d337b4c66097d5877702d3adce91aa97c1d", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/metrics.json": "d3e79e004590057c7eb199c20da0a3eef2032e5fbef4baeaa43286a385ea883b", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/codex-inline-judge.json": "7fde6ebf2c0c23b69af6fa59024b74e88bdbd5d5554ba9dbf03d17ceb91e51e0", "gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/workspace/this-work-e/index.html": "cd740b4d3bb35085bf35ea43b3e6dcb9dfec997de39ab25e67ff4796a0874082", "posthoc-runtime/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f.json": "77bf7a718200afd6f12e4d063812306b6a63f67c8ad407f557edfbd65d6bab28", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4/result.json": "f50d2e3343add0ee16ddf15a8a3c4da6387fd93a44f364f499ba5fdb2eb4c5c2", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4/score.json": "87548da2afaa7a1d4a0641fdb66f0f213cc5a2d539a3b371c6108775e0b48aaf", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4/metrics.json": "e85f9500f5cce2b92b366db7044bf5bc30bc110a80ff786ba35567febcafc1b5", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4/codex-inline-judge.json": "01e09be83d255e99cac1626c0223446edf4e275b9ec7bd8d4c1a6f0876fec697", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4/workspace/build-me-a-single-page-pet-shop-website/index.html": "061778a9c00ff40ee8e3f4bf2c683c8db8e6431280f52eeb8f23809ca6fb89d9", "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-07-48-402Z-h1a4.json": "4eaa8962ab284fbbe1f33e3ee93147ebfdf9639856e69e21cb1e159a77e7ab22", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti/result.json": "7ce5580b73214761e2a5879580fdc767b7a98b50d538d632e3c7a61ec80dfbd9", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti/score.json": "cb0bd38841c55f58a3b5364bfa40c58c053381573266ba41c6004fd2bba61c6d", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti/metrics.json": "7245d4ec0a4105f327fbaf1dc7cab5a4adf22f23139b49ec42508ba12a6f4a2c", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti/codex-inline-judge.json": "7c8ed5bc35e99366a7e93825ffa5f2aa9ad1c51549f290222ec8f566d3ad6829", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti/workspace/build-me-a-single-page-pet-shop-website/index.html": "51fc230cd00d7c72a67c627b1d7022bdf09a4178b2c645a0fc450b40885e2f00", "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti.json": "b8e96ff91e2621b43a69aef9d444c55a5606445c52d3f4e017704033761fd355", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/result.json": "c674fa95b984b9c125327febe038d918d102a967a634cb00874b74c8e95ee7e3", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/score.json": "9fcdba4df1eda5440c0ef2739ce02868c015e4d4b0934d9aa2215cb618ad18d8", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/metrics.json": "85ecbff8e02eb8cb0ea5a64055837ec0514b7dce5e204d698d81e7cb09a765e1", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/codex-inline-judge.json": "f298c6691892eab3c750b72d05e4d05e9c2b478290bf8f73b4ead4b1d631a859", "gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/workspace/build-me-a-single-page-pet-shop-website/index.html": "18882663535f9a694d9622e77c8b6c05e3c5d8cc2e769aa0969229402e29deb3", "posthoc-runtime/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5.json": "1317e451a937cee4d41c33dacf1b94088f183af539f7550a87d147d97a2a2c15", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i/result.json": "8fb549eee3f046d6e979ada57df9ae7378890572392a7e19cd20188000183273", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i/score.json": "f5695dacaea5a44eae7a65e3df5c73538871ee589b0d34b75dbaa2bd17e55a73", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i/metrics.json": "34d136cfc667ec268d46e3c03612430f86c5ab69344c054f628926b523d479f4", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i/codex-inline-judge.json": "b10089ab9ba464c57633679183b5832abc6765871a918d0c7e4f1a300cde69ac", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i/workspace/this-work-e/index.html": "4302bc1de24baa53fa5fefe81b3db80b8df67a841595e9fa3a66db743d0e9a94", "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i.json": "53de4d3214bb49b950752bb6209857c9367933c20131fbe48d12755893755ef1", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw/result.json": "fabc9b0bd7922a98c519a073e533e33e624eb565bb1e01e0affb34eaa1b031ce", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw/score.json": "b8019225cc7218aae62486c855111b357886d397e7923a406b302ac12cbf9cf0", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw/metrics.json": "657769f95747c2d9c579849a674776581b0df4aef2a0d95901daad3926bf2fb5", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw/codex-inline-judge.json": "e1539d83338190b85e15d43277591ac08b16ba5fed2e186c2e5fb12ebfb1af4a", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw/workspace/this-work-e/index.html": "8f6e1cfe7c5ed960d1dc21808676988134c161c4fc7b917003f0349ec8a1cd39", "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw.json": "aa124f0253569b8df36a2fb7a25a14406edb9981b63471e3d29e99f58964187c", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/result.json": "dffe3d86c9f10ae50c6a200a084ff72fd74d58aaa3d48672c3bbef276b7f5458", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/score.json": "f5c8e41a7fb013a8c0264194746582dd7a90ccd130d457c71ecba0e3f7e85108", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/metrics.json": "834b357d9ded43c4ff63e8f10592df56fb64e75be0a7d3a99cf7e85fe68ca348", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/codex-inline-judge.json": "27db3dc8501981cba5bfc83265bf943c033459c6a19ca70a309f2c7c20102457", "gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/workspace/this-work-e/index.html": "2c9c1715b038c86ffb62ca69f60b9ce601dc7a20b50135efc81257a36fc075da", "posthoc-runtime/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p.json": "2a3d74f6a050255f3fe255fe14831748d749baacdc1494c88e38337675fec591", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse/result.json": "ceb6c1463b9903012990b21632bc4c6e647d418db0b773f5b9e27b27db4b0769", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse/score.json": "ce8c42b39fd7646779316384bb84564357fa967c5b53610bf323f5a4f8a6291e", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse/metrics.json": "f4a60d6b8a953e01f6695d35df5f5a21fcd9a10fd66b3fceac42cbb66e6bf601", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse/codex-inline-judge.json": "d8998baa5cf809e6b285c130e9aa69996723b38db39dc23b4ff675ab351b0e1f", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7/result.json": "6d6b28167a7af4d9e198af0c09b15e4f9d5ad26652d281aa4409aa92ad5654df", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7/score.json": "a32329607047df11b1eeaf70dccb11a20c583f9c62f9cbfd61a126f48a81828d", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7/metrics.json": "f91f64c5315022c10eb33e3fe8bf88269c1f9de8302f0e877016f46a1a1afa73", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7/codex-inline-judge.json": "9899e80f631a6590cf60f423fa971ecd9c6d20616a11e41ce862126a88efd698", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/result.json": "beff2264652f99ecb28601de1b24a0be4ea28b8dfe8a264c957e951319af8261", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/score.json": "520f31e31a9e150f8b43e82fe4bc9ad4bb4c2454e006dde6a0812080fe33c7cd", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/metrics.json": "32ed08caff1b858eaba226689f76b0fcad5d54e4853f23e5219255307917d61e", "gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/codex-inline-judge.json": "d975af701eec2c8e5706455b2430d1d3f5a82aa1e017a74275a704bceb781897", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn/result.json": "02bd31d9777ef8ab919142a694557458764a2d3f5fa0dc09d867c198625f13f7", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn/score.json": "dbc291b95f564f4cd793c3c610e4794af0e84e8eea54a132b8c9827862a5c9b9", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn/metrics.json": "18926c0e56292b3bc9f0a26731627c9d2e5762b51a7b6aed15aae3fac8bfaefe", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn/codex-inline-judge.json": "eaa0d5b7dace31b420761e8209e67e6afb17b7e1a20aa8f2cffb8dd4937b4c3c", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t/result.json": "dfdd2315f5b909533fccfe132e8075d66a9b3841ea932d3f1e4030cdb0ce8774", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t/score.json": "2b6c7ca1cfe14234fdb17871e30eba6458267372b51f777e1fb4a542d8d67f8e", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t/metrics.json": "da8a1e7d2804482c3a7851d979d31549af3de65c065d9fd5319c59f4b12d69ed", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t/codex-inline-judge.json": "e138a0809913ccdbd5fdfad0154a95d87e3c4441fd17591f7c70c8a0168405b0", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/result.json": "21f0495aa17dfbc7201c4e6eee77b533023a517b35260c06fde252b1e421b564", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/score.json": "61c6dde3ae93bbc263209e506f484b5b9f1f7a91649405f6c25eb587e989e884", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/metrics.json": "db216c644ea750aaeecac7e479f4dcb205904f74d76d9d83c6260b90835321d7", "gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/codex-inline-judge.json": "bd94cb2c0060b6155c44832b4012f0bdd139e636b0ae6506bf341a1782e09c79", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu/result.json": "2399e4f4d66d4f8c86a9b8bef26f89602091ec6f3717bff78ecce925c43dc9b0", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu/score.json": "25d16140338ef308fbc83b31d1ef892ccc23a0eaf0f769a3b748b5bca51ab6ac", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu/metrics.json": "0cb991c8be5a5d0127632695171cd5275f1f0cb34b8fbf862f332610ee391649", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu/codex-inline-judge.json": "7a8e48d12af8a637dc8a8c0d0f51f05dd93b52f65d65850d524b3d16ed20e53c", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283/result.json": "925c84c3cad58533d0481db094723ecf4289229859349a7f668ce72821a0f97f", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283/score.json": "edd35cfc8bc9201c51611612d0d8c6d6b1eae7e8ad91ad4c89d4ec169d453486", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283/metrics.json": "2bb71e812327b11a00b7f16b72b323277b2060b04d8e86ca867badc9c807e646", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283/codex-inline-judge.json": "3921223ae24605d1d451baf429afac4d1058b44bc513cdab0ce960b4f72963d6", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/result.json": "46787f496c25b924ecb125ee09a85b2b0f262440b832c62204ae66732da17cf0", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/score.json": "9c0d42f251eae98b5da4e0090d9dc7064be988d547cc64d9941f3c3f42297791", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/metrics.json": "b6ac11bd2c23ef5a832d105d5d3769f71f55e63e93de09bc416bccb1728c5c75", "gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/codex-inline-judge.json": "8b182fa2a92113d396c5f3f272d4346c26dbaaff3261233492b4070ffc610d0f", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk/result.json": "e5c47f97a5beefa488f9682d395bb628273e5c707cabb4324ce69d78ddf30ce0", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk/score.json": "368c155fa84c88c64b976770fc5a45de9650c887a34754403c0201eacb470e87", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk/metrics.json": "e27ac9f67d42105c4a78ca679fb4bc80d5fb624cbd326d05a4cec2e1399d5ea4", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk/codex-inline-judge.json": "b5883d11eac01f96154422aa176ed789d490230935f40f476049600160dedeb9", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k/result.json": "b0463dab6d7b99146c6164fccd555c59847f8dac242e216411b854c4cf6bf63c", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k/score.json": "b0753249489021cec0287c96e3ebbb3280e2d80f824ff4e66b948366d1e94225", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k/metrics.json": "1ab424f667fb7ab4d557ded7eb71e3467b33c55518a00db0a21bbaaed0e59724", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k/codex-inline-judge.json": "7df76c89ff6d67683413adadac02f677c3156646922ff49e4d67de62d8cd44a1", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/result.json": "18a23fcd6929ee370f3f488ac2c1c6095cae1c7f2d17c666bc4408e07dc119ff", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/score.json": "1e2ea00aae8d907e058c93ec43f1a448c87571dbf096aa065604a017bf2147ee", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/metrics.json": "0c3e17cb52f6d17939f9f44e5c995443bfd1aff7823d6ae5a38af73f6589f0c2", "gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/codex-inline-judge.json": "b028864400d0dcf6d961b6e285980518ac99f6dcfacea6776424f7122dff9c27", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4/result.json": "9e6b81c23ff2cd12472a2fb4bb4e6c0a73c01930fc23cff86f8bcc0a2c8e263e", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4/score.json": "ae4109c89d0c1b2af00c69e738dcfe23dffde684a2835ec18f9da46e4ae4ccca", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4/metrics.json": "e17d1b24c1f0eebf3ec58be6776dcfcfbfa4644084ce0d0b44b7ff626dcb9a54", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4/codex-inline-judge.json": "cacd6cdb2d3f482a3486a998f1cc1d2c166cef52265d53193ff8af6fcd350116", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6/result.json": "1d386036dcf8c4f651230920d1eca2b1f3a58af596a99ef03c63bcd60de651cf", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6/score.json": "a8cd58c8993a9d07ad0c988227bc8135d061bcb29732176337996e58d45e361c", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6/metrics.json": "4a2dac867512391ad0b76a3064fb4be513a670a8ed71811d50390b36ad305624", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6/codex-inline-judge.json": "c6db839066e370d3d86507a508580c6876ebdd11472ff3350e648886f4d1ca35", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/result.json": "d809c134f6251e48e29101604c5bad2fefefff2ba7b9d3a817d8508bffc258f1", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/score.json": "95e0d3fdb73baa320d4f2221faa3b44608732cd5a35e4994d3f9f0f804504f15", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/metrics.json": "971a7f7eff40ae42f717c024b3c516122890d994a2c645cf5040805c0cf78a9e", "gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/codex-inline-judge.json": "0f489216fac929c43e0f452fac286b4f1a01293c84f48d74480a554013b2a75f", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0/result.json": "80a40bbd57e54ad2515c93ffaef1f33d8df2cc8d6e6e49e512f0567ddfd8f68c", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0/score.json": "ca64b34deea93758050eb00b8b9fce6fbc457b9067efbea5d91b627a02e81225", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0/metrics.json": "cd06f075b294a63caa3e264bcd106497b67b379fdbb6763ab0310f74631f6ee9", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0/codex-inline-judge.json": "3b6dc7dd1a3a1baff02311e840c6fe53639ff51e125a46b89ff0be0edd1cc205", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje/result.json": "5a82fedc41e214db576fe887a0d350e5fba9ad480f1ac686a03090049a6ae3f5", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje/score.json": "386c61a81583c64999cf0f82d4867e312d044442f5df38750968285421327960", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje/metrics.json": "9e5db68a772b17a9f44122290ff7dfa64a6121aa684aa8714cdb4d276b2faa2a", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje/codex-inline-judge.json": "a5268a77d4aeb3efcc4ead7bb3d2ed3f0c2cd14bbebd53387b2ce970b301c726", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/result.json": "1574b4ed75c9286d12a446db4153d588449af9670a147fadea8bfa1aa36f7a02", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/score.json": "7b37d4ba9c0a041ce7d5394a552edd9f93e5ff510a6292f59b4f1df34f87b5d4", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/metrics.json": "92f931fb6ee01679fd3c1f045f5af2091a4e1c8537c2f1311a95bd742a662bec", "gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/codex-inline-judge.json": "e327b38f2982053f68a802011243b6c1e63832ea706e2234115ffbfc68babf96", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-07-47-173Z-3vdq/result.json": "e64f21c674d91edf75e5ca38f21a277bf3498109d7cad5db8cf4ab611613f979", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-07-47-173Z-3vdq/score.json": "c16082fb1ed43ace3539e6e0ed7c00ef2e19ee44e436b42104bfe16e6c84b867", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-07-47-173Z-3vdq/metrics.json": "2540b9f40b18d15f26713d652ff039cd2d011561639ed3536389384c862e3497", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-07-47-173Z-3vdq/codex-inline-judge.json": "ec651f613eb4c4213fb33162a8fdc8fef65a711f3fd518953b4ea98d4af33d37", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-19-206Z-semw/result.json": "3ac25a9455aa6b5349f685efdf338c6a7e3f54aa22a59fbd2b10541d78232bf6", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-19-206Z-semw/score.json": "1bc1c3321d52f98660acbc8f735478c20f2ca4eafce8340cd11a37c1fc322d59", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-19-206Z-semw/metrics.json": "9c927d8e0b9b1ce2aaa70c73d4df4ab0b26d728b5d1b71d144ae5a6ef19f4a79", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-19-206Z-semw/codex-inline-judge.json": "214daf616a3d416ffe4b4063bee5fcfcc4f062043b30ed42cd97d1a68694870c", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-50-324Z-r4y0/result.json": "cfd4451151594fd026044a97f8b49ce5c4289530d0221690c642b47d967d97e3", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-50-324Z-r4y0/score.json": "256975e011fda59dbbb9752185b686639350434ae2ff4211efe10e663c31dbf8", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-50-324Z-r4y0/metrics.json": "8f75cc969e73d37c8a02405af4249427f4be722166ee37b9e33ad9d4771b073c", "gemma4-12b-q4/core/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-08-50-324Z-r4y0/codex-inline-judge.json": "466e693dc1ae97928eee03449d8c7d4ee1313b94db6dbe166e82db9d3c308385", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-09-24-593Z-7k5u/result.json": "2194c9e3af39874d016ef44f41867d9c7eff159ad3026485f0075d34506a210c", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-09-24-593Z-7k5u/score.json": "bd47e5a160906518dd1eedfab58bbf063d497ec737b1a46168a3332bb0cb7c38", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-09-24-593Z-7k5u/metrics.json": "acdf0e958d72dca166c7f1babb24f469387ddad0bedc82f9a41e34e9f7245eea", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-09-24-593Z-7k5u/codex-inline-judge.json": "178a43d7d72bde5cd9c9fd8a7e68adf319330a89ca6d4ba0175c059ffb612a44", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-01-356Z-w8hn/result.json": "edd9b2c4d36b59a2166e5e6bfc00a75dde6e38b762e869bff387f3a908d3e63c", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-01-356Z-w8hn/score.json": "c4d642614ffd1c2cc9e8386644f44ab3fe2006ae496de8496543bb39a0fa3584", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-01-356Z-w8hn/metrics.json": "031175035859387062578184f2bd08cde12b5c33b960e92a2bc71acf65495870", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-01-356Z-w8hn/codex-inline-judge.json": "53f588abcfac3d91ac0bb1ca91434ace38f6bd281026a4d7e3afb6195ab33eb8", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-38-043Z-rol3/result.json": "a51ff77c870e35efda77d093d821abd2b05d2ddcc8b5fdc373e0c36b8bb4c6ee", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-38-043Z-rol3/score.json": "0888f4552d06b6e0d1bffb8b18aa50f7e6efab00a36e1eb38bb41e295f08b4a0", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-38-043Z-rol3/metrics.json": "6413acd269cb7c756dde5a92c5a4023d83e9c35f786e8ddc3571dfb6fa638352", "gemma4-12b-q4/core/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-10-38-043Z-rol3/codex-inline-judge.json": "c518c3e2cf18ba066d49349806505db25e94b767b629b7b9a4c985bf81fddd0f", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4/result.json": "38f54aea34e8e09d4eb8451b1f9615aed6514b499ed4faf152f65c896e69265b", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4/score.json": "8b3199e448b17aa10d4616dfc94c6b4e9c59cd5bb40a198c698592e716004f71", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4/metrics.json": "daf9ea112ee48f8baafd2643d22b62860af4bf516caa2833a76d1eb5e27793b0", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4/codex-inline-judge.json": "e1bb8e5a48d097ca4d437f7cb29f5ab5f1138478262ea514ea4f4bbd90e305f3", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8/result.json": "012ae9e6e7a499c26bac97746bc5bf7720729a2f7864d373c727a7bfaf70f631", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8/score.json": "f94a96c4222da19bf8acd656637c9bcf5036dccc04c2337911a2bd25fdcd550f", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8/metrics.json": "276892ce648305e766d705ddf9a2e0755034f579796e66dbaab6893866045383", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8/codex-inline-judge.json": "2f360d9123394739b1bb4d966976e65fc2150875125cf9f439d717e30748d633", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/result.json": "ba2714fe606304843bc1ed8c85e639bf970ebd3c9807cef12206ef327fe1dcc1", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/score.json": "3a805034f06a241be38f5bcfc763ec9677b261af50f94aadc0927426bdaaca25", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/metrics.json": "565f01b73a7feb1d3dcffb71e2017316979a62a89f070b206f45a25d2449342e", "gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/codex-inline-judge.json": "f278384821e555aed9e3e2daf4a67350eef019d9377d14a85f0f99764cbbfdb5", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho/result.json": "0243390400d0d116144e8069a07ec06525b70af60aa4d55de49f42c5b6b03117", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho/score.json": "5778bff591fcafec2311b33f329a56b53b0c295e48700a48221158c56ae010bf", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho/metrics.json": "b54830c0cadef5ec2d3708960a8db7065072d5e96f07a5a586d1dfe507c16578", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho/codex-inline-judge.json": "1d13130dac6238e3d4f1140a4cde23a374eaa0ca728269ba3142a262cb2045dc", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i/result.json": "7b06638af2fc19ec7cfac21d2840177f5732ec561535ae1a72914ec0aab3b556", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i/score.json": "7155ee64d3efbf6c523fa017be2b24b870d107ebe216c757522a5abb8e219bcc", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i/metrics.json": "ce227fe70cb81e8e029ce097ca487d7d4f63bb07d0e72436fe7e4300068892f0", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i/codex-inline-judge.json": "43b63a2d00e36f266a8977ee8bc2499902eb660702e4641387b67740b0e4f234", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/result.json": "62560fb737cb114069e80e9454f8c1e420e9d8f0eb5a6ce64561b2c6cfb1bf63", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/score.json": "08c46d5dd1dc38b0d1adad0a53146f063eea9d22d659393221e1a7ed2c9e27a5", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/metrics.json": "204e516a06158261e0c50a33e485659bd7b526ae34e658f93b6f61d1d34b9e28", "gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/codex-inline-judge.json": "e623b0a062633ecfd62c86914dbb471912aa3602635b8bdabf497f0b0db34f29", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri/result.json": "c027913aefd828be739a699c57ad4ce251a1cb67c2eb0ca8c3ca6ab4ddcd63e3", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri/score.json": "5aed3486d7d79f7283059cdaa2a98b93fc9121a55a353c0ce2bbe9a29b23dc08", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri/metrics.json": "5563f642a23a2e15c717907678661463ae9d77b1693d38d24def169158363c33", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri/codex-inline-judge.json": "201d08eb64aa52e52f67bfb3e60cec8c36ff82ab4684b852002a9f4b2e10ff43", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m/result.json": "0700d51419badecb39272770561aa8652ee9de546257cbe043859e7addd503ff", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m/score.json": "0d7b74f7effe607ac28a7145cd06e9e7ad2d3900d4e2cb693423edda6512d33b", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m/metrics.json": "957896a4081cb98fd0d117abd6d0d6a71e26fa168931dc162bb56381f4c89b0e", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m/codex-inline-judge.json": "8485c9b1f9b3f879da1c7bbbe898eca86dc69eb6b29c149d5ff398d32fcabe04", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h/result.json": "5e62c5a77a6c26e01b559e38ab0b425605793390d12aed7b7c3b81b053f8a5b8", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h/score.json": "0da3e6a34e1fc42dc90678517f3277669e893cda4728da0b5d8bfde645cc9229", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h/metrics.json": "9d580412bfa2ff14e0e4a81be4939250aca48fd58e9845716543fb15e2c62b45", "gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h/codex-inline-judge.json": "556fb2276e1755fd282a5bd7351665c05402f46fd1fea343c4cb196c1601c0a2", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho/result.json": "d2a1b6929ce9b5302368eff8c8bb2d1af54c36f4b57e454a590b1e18b1d128a0", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho/score.json": "c7732788d53f3c8949c0c81eab5a9ffa2b8b65bc3d2d41d4877f8f0cceca0c38", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho/metrics.json": "071551f76f998b5ff77cbfae842dd50fa3eacd048100c19ddb01d0e6d4a8c0ea", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho/codex-inline-judge.json": "a1099d66abb9cb577d9335448294302d6a0eefbe8316780a9c071d63f44820f5", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho/workspace/orchid-launch-follow-up/meeting-brief.md": "a478d5fd8377e3f898204429254132d650c199eb563c3e074cad0f1884d5a2bd", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9/result.json": "f2ad0b056a8cc8ef372eb9add98448a659f0755b9553739db4112ddf07f51b02", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9/score.json": "056294a94f9bcba11dab1ed88d259f3a274451eb01303ce2327329bdc7a8ad00", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9/metrics.json": "c54c58063a3c812c863441b4c2bfdc522f650988397ea7c2613b30a2613c9953", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9/codex-inline-judge.json": "3e0c9e8364524ff197b73c0771a91dc22fec2eb12a6c4a6b70c0db857847f93f", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9/workspace/orchid-launch-follow-up/meeting-brief.md": "e43118cfd1c08721ae906cd58f3a9037c0e4b5b12547a9b00f3f41a521bb8664", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/result.json": "9fbb829e9d366b39888a66c038a15e4e9adc9b0ee10a2ee3a69c365c205bf5d9", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/score.json": "24f1db6fe4439e53ac4773c42d59cfcf097cd122f10c8f3b3fbf4af8ca3be6e2", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/metrics.json": "50bfe7eabdb4b7cb9c6466f8eff8362bacdfd6efe3cd33d63406f9b6a53b9351", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/codex-inline-judge.json": "0ae462a0661cd552b4c32029e5c14ff6d14f8dcfe1dd08cb90cd7f9da0b9ee87", "gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/workspace/orchid-launch-follow-up/meeting-brief.md": "77dcd0b6c4a4f8aa01577c082a58d9ce92597e2f10df90d6a05403fe9761ab4c", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc/result.json": "e0665c8024558092bf71ead88331aa886a0c86e6772cbd52a0e79c3a08603d21", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc/score.json": "04dd50266100c1044e355509accc8404f8abb153d83749cbcdd0a5c800428770", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc/metrics.json": "66b1dfcf8065b33328c0aeaa17c5d4fd7457d0c2c2d8356f1ff4a885709ac78e", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc/codex-inline-judge.json": "383a47e5f1ec23a80c4fc89946edaf8108e8f6915ddb2884b9b4d1ae49ef8605", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md": "bc4f69b924094af548ecbb098cb9ad29d162a04f9a204ce6e1c5ffa1ed37c689", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9/result.json": "d8d91fbe6756c5d2491d99617a17118b7dc4470871066e9181e25dc48284a773", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9/score.json": "fdbea5409a165b666760ac0573cca62f6c88c822a9c1fd34d751e94787e781fc", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9/metrics.json": "4a2aa5801021915cdaba3943f67dbf777ab817e816f6d6e7400631abe0e5bf24", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9/codex-inline-judge.json": "342b6f38a2d7aa5d15ca0180edee1c590833c02b336527cc81ff8c3649e36db5", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md": "29a38967931d8777b83745e9e353c6423be054924a93f7f5a7898a5a79255177", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/result.json": "d999f2375b0940242f1958462cf6db954bf033cf8e0ad185a8b17e0e92bd49bb", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/score.json": "7fe4e1681dc7eb0c9b79d9bb7343294817b8474418db8952cebb0fa4299cbb54", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/metrics.json": "144278fe35a57da790cab7cfb6826bdf1de50a67b76c171c234ef97952750c87", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/codex-inline-judge.json": "228f4080a943e87057f39292d65b215b1460df418c6471cb889ed950af8235d9", "gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md": "b173a4e36d44d0529d3bc6d84a42f63c40e30a0ef4a1ab8322180d9c8ef0dfa7", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt/result.json": "6fe05bc7598331826344510ac044efd7349d2f706784ed8bfdf062920e415a11", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt/score.json": "90649e2cea5bb26845f16f50232975bbdce99cfe29a8ec5f0bd5c89e69fabe46", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt/metrics.json": "8968de1957ac27494ba1fac009e788fdb9ccc576f14f53c1d97bbaec7a6025de", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt/codex-inline-judge.json": "cdea3858684c15f3a0202e4556df0702e0b6e2dc6e24c1a074ab9fcfd107aa9c", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p/result.json": "6391398f4e2bade976180c045af31e7d40505645de259ad9e560ac1c0884fb4e", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p/score.json": "e5dbec329de1ee447244e8635a931b34c71c745a259f6709c5f7c2f35dfe0c25", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p/metrics.json": "f90564fd229793d4bee3854f2d4bdf79d67d7238caed5d7661835a786a4976c1", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p/codex-inline-judge.json": "047eef9ab3a21c0f3754340f3b9567766b5b9dd0478a8aca25dfad4c5672fa35", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2/result.json": "2bc1d04bcab9cb259d22abbd9392c857d568ef4ec7431a706c39189608954c2e", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2/score.json": "7123ecf6b76b0d86302728e17ca718bcf5c0be65373436ca1017e7a1f850aae1", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2/metrics.json": "7fc7ce5f7e890c64403945992a91717526f4046e3f1ef3d170f2c579c4da92fc", "gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2/codex-inline-judge.json": "b49c607f18f1f7d81856ffaf72d7e1981eeacefb7fbcf26c2e12f25b112147ca", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8/result.json": "47309e634945e18534b5b9f3a9f3bb3f8a0260ebcef5ae2a95aa0a2d8f50ea79", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8/score.json": "3f584feff5846b8acefe1635b495e5d87bebc3ca50a10ea45e45296f41a109b0", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8/metrics.json": "07be510fe1cb62526d7d220791265ff083f94571e9b4cbd27714ffb9a5c8fb95", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8/codex-inline-judge.json": "ce31b99aa4ddcead557fdef712ad3e9e3f5e53fb256e8f78ac18af334c8b0996", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8/workspace/brand-theme-alignment/theme-report.md": "8e78817a526957de33b77122d96cd86b7be099edbea958ae68e126156e62b68b", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao/result.json": "4f2b67c779cffc3c2d5f84cd062cb3b15e165ba4d930b865394ba27a9c0af332", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao/score.json": "8227212aac8705c87dd1e665a4bb826d2950324444c5cdac0836fdfb7cf30eb8", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao/metrics.json": "950c671b92a1c0f0d204390d8457b8e1e84d8308705aafddc989c548b8dacbf2", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao/codex-inline-judge.json": "c42cfdb4b8d58486d96485564c84e87a31fa086dc8d76b962b905938407c8d24", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao/workspace/brand-theme-alignment/theme-report.md": "ec4ab0be64e6ce7d7f9a1c78d39eccf0e2f639fa6dd467724769c3c4ff48d827", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/result.json": "a916b2157872ed1b3940cd95a53ac8a4a0065d1579578f4a664ba0fdfaa6318e", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/score.json": "dc7753cd284814c650b61f998bd7aef18e1bce49e81d6ca7ecc7d7de4edd2721", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/metrics.json": "d4618ec89ed75a1a7887e667e5470ef900269edaa9fa51ff27e9d6dd29649eb1", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/codex-inline-judge.json": "59626bbd70fa7d10f10d7ef41861442bbdf284fad983db33039afe7647bf4f56", "gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/workspace/brand-theme-alignment/theme-report.md": "6b6c5267d7dd43a0fc23b124d8276e0b076fb77367f149695619c22e61e69536", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/result.json": "76dbb5743bfa26954d84c7179dbe08c1abbc7ac4d2fd050be73f03bdc32aeae3", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/score.json": "b65ce5e68e4f95af446977cc6b7513003bda48888a8b73e9aac40103bdb57db8", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/metrics.json": "dc6b526b2d82d2bdb9acfcbe25e8a1d760ecd965741423783c046f663f9d1a12", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/codex-inline-judge.json": "b9fb8ec3cd565a04ed387e9112e690d42d3054bdd213989d52df89e0dbd9760b", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md": "e357e13fda5b1c583b4b0b0076484e7c3bb696afd9b229c9e50351d89215913d", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/workspace/powerpoint-grounding-eval/source/brief.md": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/artifacts/powerpoint-grounding-eval/tasks/1/sources.md": "9b80f8b560ed2d143afdac56196e97ca2748334dcdc14f5ddefe3de08ab7ff27", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/artifacts/powerpoint-grounding-eval/tasks/1/outline.md": "92792dec74e4271fda33adba4fd0afbda07de9ab838f35fa439c612f6ebb1ee5", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/result.json": "67f0477e861c340f1754c614604e597a28b36565e50902bc5ccc9d537565c29b", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/score.json": "8b55982b76d8beb56bc295a5baaf2b81b394471e2d6d39823ac6de38b503bcfb", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/metrics.json": "11f26fa8175d9f4c2573bb71818c899798e5a6543bfa1ac2d70c3fd0e5e70eb6", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/codex-inline-judge.json": "afc923d3d9b6f4b3edebc783e75a974019a97b95aba8ec3eb0136790fd399e4b", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md": "68788bb4c73132cb4f9fc425f84bdffce09d6d6939f203255b3d3a5f93874cdc", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/workspace/powerpoint-grounding-eval/source/brief.md": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/artifacts/powerpoint-grounding-eval/tasks/1/sources.md": "715034706c8d3fca4e5c23ce7a8b0cb623dc9409d55941dd65c8af99fa01c36e", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/artifacts/powerpoint-grounding-eval/tasks/1/outline.md": "460a8fb5497d743999c5f1eebf0bfc582c9cf3b1013f85386b7ec048b383d6b2", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/result.json": "e3bccf943060d7c30f7de82bd4150626afbe4b29c4048bf20d497f94429a5ad6", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/score.json": "e220ba815e33c50e3a705423c1e62554b3c6f6d13494e6ee3f4898b037c0f7b7", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/metrics.json": "bd5352d4b4e35afeb0b5a611d2d82ab73d4864fb98047c666b16a3d8bfc5d905", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/codex-inline-judge.json": "2cc643e887078e9f1703e0de029f58376a6d3a55fc68ddce98557a61270103e3", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md": "bfe3b9f1d6cedd9446b2e17109e6f7ccc4749664e4390b281250fb30374b17b0", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/workspace/powerpoint-grounding-eval/source/brief.md": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/artifacts/powerpoint-grounding-eval/tasks/1/sources.md": "3ca3131edb37bcc7c8e6a8ddeae185005c903468a997253282cee22436aa4033", "gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/artifacts/powerpoint-grounding-eval/tasks/1/outline.md": "aaac4a3317a52ec3d6a3d316173e364cf2b9bfdc74f480bee9d0a4cd7dbc7838", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729/result.json": "c47935fc281e82fb73eb9574d5ec83936d508d98da9613ce98947aca0b45f432", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729/score.json": "186b92aaf27dfdf55f62bca2daf7542852728b144544ddd661312147e5787792", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729/metrics.json": "0a7c0e127b89132c56bd03ad4001c278e6c00415aa652a1ecdd936701f4eb5fc", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729/codex-inline-judge.json": "b0b349081695fac958336a45714e78c03f450a5ebe296b9c73a9d3d5edda9d14", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729/workspace/analytical-engine-research-brief/research-brief.md": "a13fbcb1a918904c4a496f47252d2bc852f1f25a510aa4cad6f34be66354935a", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729/judge-context-wikipedia.json": "19c1f142303c9d66c754ee825f23c09a41a234389565a4c37ab1ec40ee58e7be", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc/result.json": "0e158ba52c63a459bd2ea9e3bb23dfa8ce7f287e9f64c52a5c6de108f1acb14d", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc/score.json": "d2b58393623a8cee59b36907be7b052a9343eb33b5e165e41120d392d507d2a8", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc/metrics.json": "52302f76a13e2783674c5b530220d8364a24de75fead98edcfc0465997fea748", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc/codex-inline-judge.json": "5d38d3bdee9628a5b681fc6141d0ae0744454ac6386dc0e7be17bad870d1a8cd", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc/workspace/analytical-engine-research-brief/research-brief.md": "f7fa00c73a115754c796058e702721028bd827e3331d691d02de28cb0dc47b91", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc/judge-context-wikipedia.json": "7291802de1bb87c73b4f0ade731b9a534360e25355572c802e75a8b8e71bcfaf", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/result.json": "f1a46971cda5f0f75f2c5562af3ef31dc0a78f0fdeb76fd18de9bd7a6b44f0f5", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/score.json": "1393a23a2c4d2224ad77e12579941b28b68dd3030177d31441f54e1fdea85556", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/metrics.json": "ee4fd16b8aeabcc011a645b9e2c71d00ed45b43a0588baad08cac267c7c477b8", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/codex-inline-judge.json": "276431016f37484bd08934ba2802fe10485fe25b3ab381f55d8121aebe1293fa", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/workspace/analytical-engine-research-brief/research-brief.md": "28b458ca370846c4962d1bb9002bdf86d493bbdf04244d774e0ec299a0849258", "gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/judge-context-wikipedia.json": "da2e630c240cb8bef80af1ffa2e8f0e84415825816b0df354aec4ed5ddc63da2", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud/result.json": "ee24f5b9d2f2049d3a511290328a61c30b48d7c491baaccaef7adc4581211b09", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud/score.json": "a1d28521ce681869173f9dc752ea0d0b382ff5c405fd996cdcfb44d7a9c3eb96", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud/metrics.json": "7bbfb6078e032ba74aa96dac76552cbfd964548a853a306230ce32cd8c4b8192", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud/codex-inline-judge.json": "c361ad96d22d4e0b956544551e86667a52a1ef449bcd772c8c2b23db428b9f92", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md": "ee46ca6e4e299e27aad357b140bcfaec06fc8c3b3d39e30e0d669c26669e1098", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud/workspace/calendar-desk-week-plan-eval/followups.json": "9738e165a80063b91a9a70de118e9b94f4d8d50d662ad1d629c9d01987421757", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud/judge-context-calendar.json": "d44deef37ba4b21e803349992506ec11351be18eb9ed9d2238e1b8ba21c95eec", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/result.json": "b3dfa118a6db5acd9aea8b4e8138d31c24b87eeadd9805dacf0bad2be4466382", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/score.json": "bec9a73d2ba6285608861b34df80405e51cf447a4b086cb1cefa940c381f219c", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/metrics.json": "bc2bdc792c35e6d57f71da405907b0ee0ea02cc4ae38eb83f81531f84550626d", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/codex-inline-judge.json": "2e87a79443316279154a5dc4f4d00bdea222ccb15da8bb7266c33895563d8479", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md": "222a0709e8480ed0744569df7956b7359ad52a94c11a01c6103c480133996f25", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/workspace/calendar-desk-week-plan-eval/followups.json": "9738e165a80063b91a9a70de118e9b94f4d8d50d662ad1d629c9d01987421757", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/judge-context-calendar.json": "959d0de77c9ea6c1e7c94eeec235ce7ee7ea7d675d16371ee4002ad31fcc28cd", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/result.json": "c2906abe39e9de24fcdf6c1d55d70089863bc25240b6906bf3e250550a295593", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/score.json": "ebc3e039b548d40468a7adc68428009f8b8aea0c198c701ff6b54db26e926480", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/metrics.json": "1fe5863e5469b5a5f66caf4d0bd29e01c7c042585e8c287eb6b9f2c2f69371ec", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/codex-inline-judge.json": "cd6a2d868b629907d5da4b7977533adaf959eeea19e71e4b7781e79e97cf870e", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md": "d825b8892b9a924e38e477f693a56000185fbbee515f210d1101e47de45749e7", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/workspace/calendar-desk-week-plan-eval/followups.json": "9738e165a80063b91a9a70de118e9b94f4d8d50d662ad1d629c9d01987421757", "gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/judge-context-calendar.json": "9ece25149614990ecbdf4324a3b9c79d47b0ac7d76264b32ffbff946059edf72", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/result.json": "03983b71ff59693d3d8f80bcdf766c6db426e513b5e9047b3ee09ad5c5976909", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/score.json": "5cfc490b95566c8ee57e415cb338d3e5ff66b0f190810c73af6c2eef9def53a5", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/metrics.json": "730241055ac2bd6f590b64dff862ecc67ae6f84bf82c541fc1e2a957cb57c41e", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/codex-inline-judge.json": "54515d03a57dff96e2dd3e0b70ddccfdd3adc04ee3bf0b701541eea0bc33b943", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/workspace/a-b-test-readout-eval/tasks/eval/readout.md": "a2e9eca6ccecbba451f318b33d943904cb0b21038aa3811a5312e33800921045", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/workspace/a-b-test-readout-eval/source/experiment-brief.md": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/workspace/a-b-test-readout-eval/data/checkout-test.csv": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/workspace/a-b-test-readout-eval/data/stats.json": "f2d50a862f1ff5922b62b62ac4883aa4e19ac25000d3df6263c5597a84de8113", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/result.json": "d56cb760a4f48ac96be1cd6812562b6c2523c21feed26b27c620bf263a064fb7", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/score.json": "9d51b0e07a945036b62ecc27118df82be33c3aa5dfe8532cbd2f47aa89c73787", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/metrics.json": "18b79d8c13d32dea4e92f4da2d11f11e9835f982d66c81d58dbaf2403689540e", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/codex-inline-judge.json": "53bfaf5154013f6fa8a371d355fcbeb077914b4bad23bec79cb516cb43f0a87e", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/artifacts/a-b-test-readout-eval/tasks/eval/readout.md": "aaf6fbfb4dbf13bcda54e2898bcc7af4315be8edf7f6df651200a2771a460d4c", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/artifacts/a-b-test-readout-eval/tasks/2/readout.md": "c748302a9ee4b6bdec5342c0c10667f9769e4657ee591597a0c40ba15f7cf7f5", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/artifacts/a-b-test-readout-eval/tasks/3/readout.md": "72ed4f252e6ebf654072cb07b6f695474f0d51fbfea5c561c17247feb166dd5e", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/workspace/a-b-test-readout-eval/source/experiment-brief.md": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/workspace/a-b-test-readout-eval/data/checkout-test.csv": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/workspace/a-b-test-readout-eval/data/stats.json": "0d470a2734ff27c0788ab2cfc159b89d8770d198cb6b1c119214d19773bc0033", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/result.json": "d73c8de62fa898163ea554f524906bcc7429a96bb0add79a77dfd4085ac5841f", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/score.json": "eba918496b1a4f4e175827af5cfcff260d60e0d4646b2409d9717535b7f2f7a1", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/metrics.json": "f3136b1d5e0c7106edeae25e161c40fecf3080711f7307ac2b7acdb66b5c0851", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/codex-inline-judge.json": "45bc28a2d709aee923914e179ca99f3d0cd587c61381e9be667d29dc6b5de25c", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/workspace/a-b-test-readout-eval/tasks/eval/readout.md": "f38559d0036642f1c703e92ec74499731df8467a47bf89c722565f2d2b4d7d7b", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/workspace/a-b-test-readout-eval/source/experiment-brief.md": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/workspace/a-b-test-readout-eval/data/checkout-test.csv": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/workspace/a-b-test-readout-eval/data/stats.json": "f2d50a862f1ff5922b62b62ac4883aa4e19ac25000d3df6263c5597a84de8113", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn/result.json": "c3d638888e0e1aa345f8e70beca18a2cf7841cc0072658d6a5e4802752ed8528", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn/score.json": "4c713dc428dc66d3b70c3fd5472516464c9762ed0e8470f21d9253c56d7299a0", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn/metrics.json": "5ab2d482ebfbf1a0a6c366e2406191b915e64658f7101b7b34faf7928a8b66c8", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn/codex-inline-judge.json": "7b743ca621c8d3073628d4d94fbea3dbc236cd57c06370c17f47d01afb5a24c6", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn/workspace/annotated-bibliography-eval/bibliography.md": "e33d53b9808dea6017783a88fa772fcb3a75064bcb9df51ce42be2e3220077a8", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn/workspace/annotated-bibliography-eval/source/source-cards.md": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts/result.json": "563fd93262ff94371e73899a763da987c1e6390f51ae7b7b577138d0e713fa52", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts/score.json": "5215b66b227a2acd6aae9d639fa6374a832bfe47932c9bb0505624cf5ffd9f85", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts/metrics.json": "2e703efa1bd2db7c9dc4ef15a03782b2ee16d0b727cb7f3eabdac43fcfa8f2f6", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts/codex-inline-judge.json": "5bfb2cfbff1e8801575617643180d2b2a0fde8965b7d6aa951c329f8192bfc8e", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts/workspace/annotated-bibliography-eval/bibliography.md": "b033db0715fc2fc3bc0417e88fa0fa1f7978dd068efe6c7479e84e6092e5ebca", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts/workspace/annotated-bibliography-eval/source/source-cards.md": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/result.json": "6817b2d518aebf9672e759df9a57fd17ab8379fb0aa00c116d1b77d3c0232751", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/score.json": "ba541f03375e0c28bfa729d5025980521d1a2dccf0151a40e87b4044d488e825", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/metrics.json": "d73f9e269cf866fe78f1189e09aa77c0a715d7bc8e54a747cd7a0eed44db46f1", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/codex-inline-judge.json": "152b90bc05d87b22d3d8be99460748a1714fe302bf84a31a2d2c6519bc013bd8", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/workspace/annotated-bibliography-eval/bibliography.md": "5330becfbcae1b2a497ee8fbef127416db53aa7fa5a92e09badb15e1f757d645", "gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/workspace/annotated-bibliography-eval/source/source-cards.md": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0/result.json": "0ef1023c69ba9fa620bda559b84ed6ad93d14a0961f4e66da23f2fdef466015d", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0/score.json": "d131c2d536dd3f21e5f7aef68fe6a58324716918193ea2719ae27fdd77b0e2d1", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0/metrics.json": "821ab9217e52b433aef7200618b3f52770c8235b7aa63857387ae682b3f11b16", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0/codex-inline-judge.json": "6f66357668b8f3d98ec39124b4b32acb578591480a717a8169bcf060bf45cdc9", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0/workspace/word-document-from-content-eval/documents/eval/report.md": "ba577678cc81326d6e368821211e7140a3e13c6d82847dc3adf885e84e764c65", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0/workspace/word-document-from-content-eval/source/brief.md": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8/result.json": "46708580c247e5af5289d9c3974429451832be2f0c35d761422d4367376d5a06", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8/score.json": "d13e3e8a471fcb4ce4305ffb31ee60015986a9e2d5352f4bcac41486dec47058", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8/metrics.json": "7a979221622012bf9b02d7364e9b4e46eebb43a804f379f565b9d3c09c28e71f", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8/codex-inline-judge.json": "8b32f248d4ab00f3aa280b2071143e7dc1d9a6c4aceead4d0a9ba599d01d30f1", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8/workspace/word-document-from-content-eval/documents/eval/report.md": "5e6c0262e9c07beb3fd102cdbab68c6ce55d96d4d450654fe6fa8974ee8910d8", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8/workspace/word-document-from-content-eval/source/brief.md": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/result.json": "acd30474bb713e1a869219a25b39ee8869dbe4ae209620ff36f3bdd3605c084c", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/score.json": "e961a7060f3f74c9f860318e3c5e25d9091e391123e44b9f9bbf9700ae4b3a7d", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/metrics.json": "06f2fbe6d9d7e7061fe5dcaa8889e67a28b99b7a8bc536e7e8bbf449d1c15d78", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/codex-inline-judge.json": "53fe99cff4f3a81c0990593ce8115b2e575e78c848b355ce7c9ded6c37304def", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/workspace/word-document-from-content-eval/documents/eval/report.md": "5fe096339142c5c5dbc8d42932e6dbfe125039a2542de2673b4a6dfeff668630", "gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/workspace/word-document-from-content-eval/source/brief.md": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx/result.json": "3afa2fc9025d9451cbfd21f622c7fdb66f0cab420d24b498627fb29a0302b66f", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx/score.json": "60567f955d1a3b3f26b2962931c239c0a3f46501a73ef57b1cf23ae631ea95c3", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx/metrics.json": "c36f488c4817d6ec9a45eb71921f5a0e92d01d4ff9c3dc2db9768353328c9106", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx/codex-inline-judge.json": "dcdd52999f3f7bb04a5c3c83713df25bebfe30d3d4dc2f5b966b51c779634076", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht/result.json": "c23a56ca8f207cb759f97ef6abcdc7ec850df1fe9b1555036e19c50f6749b148", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht/score.json": "9b593a7328bf57fb46e88a810fa0fefd3f50f300630065127bd83447c5640d9d", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht/metrics.json": "67e620cd1fb1494411533afc8cd56765e5c8c2a5f358037c2db0259533dd39fa", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht/codex-inline-judge.json": "6d84c74b3c6d4892932c6da51da57bea36e27922ef4ccf94a3c9c0390c734995", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/result.json": "3277026b786fb9f0c748012849528ca5c5f64686059fc6c52d7624dcaee71194", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/score.json": "f2b92cedd390f3ad17f2c0b228c9d55f7aba9018ffc8425b7c3ff2ed48fe96be", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/metrics.json": "e2ed8467538fec2cc1f83263326d931c70f1a0ca5c552fda87beca37b3fc99d2", "gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/codex-inline-judge.json": "66bf2d35ed95fccd3cd27a0162e18dc9f9a8e1f6bdb3e3b4257341d0ef9c12bf", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9/result.json": "99e7f27efc8479699afdd7cddc290e85a46a4060a354c376151c03c74a704d3a", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9/score.json": "a73caaaf840060e6f7453cda6bea7bdd607fd5434eee2e2b4b42da18296588b4", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9/metrics.json": "4f4af9bb89f83060acc6ab28ebd2a6fef7f130e70bd8e2694f961b76071fe8f1", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9/codex-inline-judge.json": "389f78c0dea8fed76ee2f9c3b2961771f0869a88a6dfaaeb3b395390c7ba28d5", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n/result.json": "eefad9d1b6d994eba09b54f1d7e4e07e9b6f96940009676b27402a08ea95b3ab", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n/score.json": "8b97ab835410b7a183f2ef9e299370b2906418f4c5f8c5a502822a6b45d8455d", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n/metrics.json": "08ab522e86b21901bb82a2b200d533b9237886ec8463d7546a2e87fce6d177b7", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n/codex-inline-judge.json": "acb04ac492da7407bc5ca3f10cabd614c30e41bec055507d3296ee563f47bbdf", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/result.json": "9568205d2554101ed213a2c89dbcffb5a295dff300b2b5b8da8cea38f3a22c79", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/score.json": "e6b0a030914915955a2588d3ddda2ea3df3161328a0f9684902791ac5cc460c9", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/metrics.json": "b85913545dc1580151e91f5c022a470d121d5896a1fe55c8927d2a4d3d3607ab", "gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/codex-inline-judge.json": "1015614e18bd71cb65b40f5b007831504b13c0693497ee4c77254f48c3a7122b", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift/result.json": "072fe180ecdcf9312b2e9eda497e86f6a567032d279a9cb6aa43e41c33988806", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift/score.json": "a46e4e961a2a32c2b31f276ff3d29d890f2c7aad08b64ba0443e75407434fc61", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift/metrics.json": "6c83d6bb7ab1a2690415e7cb1d0e02f1d20dff3494345f9da2bfdf8b8f0e5457", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift/codex-inline-judge.json": "c5730c2a244a0bf5f8e0db2e620275fc27f623c138f26119f957d8bf12b17249", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw/result.json": "87c51ffd4c2287e05f0b597b04ad545d4fd56bd928e3d1a4bc111ce72cb11ace", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw/score.json": "22ae11a9081a948396aa11204187b94d425c3f75dde298af31bf9d3bab5b403c", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw/metrics.json": "b9270fcc3c602a39a0d86cc799ec46eddb6347e6c732af12fc0b2495eeb410ac", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw/codex-inline-judge.json": "c0eafe51ca91f094ff452497da43e4538217dbbf072736fbb94cff6aa57b3701", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/result.json": "a4d2a96f28ef1ac7c9bee74b0dcd0f8f063e6a0e7f391e904632fb4b4f7595f4", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/score.json": "44c096e0b88f28d0c95466fae6bf1ca354b73fdff4cb9421be7f0518dd030f94", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/metrics.json": "711b914340390393dcf55a825100ea3040b649996e3026e9901e81632a4765d5", "gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/codex-inline-judge.json": "f9ce2fd264b1b8b47c2675fc7d2ce9971c598e6c76b4c2ab2c3cbe421c81d9ee", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/result.json": "a45e22ee2632f7f19cf15c9f963934b199823c0acee0f1271e83cdaadb357177", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/score.json": "bd90a7903169451e3c9010df0540d859e651d1fb2149fc5cbf5f96a572029ea2", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/metrics.json": "4b711f863ead8649aedaf1a0dae995f8c1f88414337564da3e0a2fc7d9058634", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/codex-inline-judge.json": "90fb28bcea7f56522195be44f0c27e469b54f0bf77a64c0c924ba9ef8420d4ff", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md": "46f83785a6d4f1f42e1285c47cbb7fcc23131f0d55ceb16b91f11cfa2193f277", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/workspace/code-review-workflow-eval/source/craftbook-eval-brief.md": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/workspace/code-review-workflow-eval/src/payment.js": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/workspace/code-review-workflow-eval/docs/review-conventions.md": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff": "a1744bbef7b2c7cbd7ab3feba3df59a32a472f32781773715a67b8525758b728", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/result.json": "1bc884025e7eb5097f5ed97e157a12b42f457d9ccac212dde1560ebcf4e2b1dd", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/score.json": "e7ad990ac8ac6862095bcc36b43993d9e4d3e847004b65e0ce391b2024d0fa47", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/metrics.json": "bd9dcee0760114b5c960b941e5523b7d9a477c9908aaf5bfd5c1eecd5382c6be", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/codex-inline-judge.json": "56ce0ea6bf5fef632190bfadfe5fdcf42d2c4f4d2ad24732f3f2469c2358ac2d", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md": "c114a51e8c04871167eed0104627d6808b4845425048a41b0da60a6475a56b4e", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/workspace/code-review-workflow-eval/source/craftbook-eval-brief.md": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/workspace/code-review-workflow-eval/src/payment.js": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/workspace/code-review-workflow-eval/docs/review-conventions.md": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff": "a1744bbef7b2c7cbd7ab3feba3df59a32a472f32781773715a67b8525758b728", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/result.json": "cee29fa25016977aa1e7de0955ed9fc542ef81f5e2cc0753422259ff93987cff", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/score.json": "55b11eec9c0ef99a3742258dec29f6e6edce3d2dbb50d853846703598078674c", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/metrics.json": "1950815fb942385e784c1ff91f286c3b5ab4e62a72cc27309601ce0b084ea824", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/codex-inline-judge.json": "0760af5dabd641dd4de76a2064bf04a41523d8046e80569526e7f869d684731b", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md": "ddb1f84d7675762fe84648fe1bcd6401039bd791f68fda1ccebcdde368513098", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/workspace/code-review-workflow-eval/source/craftbook-eval-brief.md": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/workspace/code-review-workflow-eval/src/payment.js": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/workspace/code-review-workflow-eval/docs/review-conventions.md": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff": "a1744bbef7b2c7cbd7ab3feba3df59a32a472f32781773715a67b8525758b728", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/result.json": "127d54509e433560c20382e64ca1293679996e3c8a4469220442228922b912c8", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/score.json": "a21f26f7ea57a4af4765d9ad02edad1286572f99c365d20f692b45477754f8e9", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/metrics.json": "75654adf9d62ee18179a545864f1e1071eadcb038f7b3ebb7467b2e913929641", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/codex-inline-judge.json": "7195bca94eab794679a0b9a355ba51e6138ff40878bb1ceedc176788b76b8863", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md": "6b9b071b48e5b0b52fb454231f77757850957f2fd10dfb2e46631053f7f04942", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/artifacts/api-contract-review-workflow-eval/tasks/eval/review.md": "5b7ab88aa9ca328a73b0d0206f5f09f4e1c2b1866f8b3642677fe672c26c7686", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md": "2bef943d3844b0d8adc402c01b763750ce319462b2fc4cdad69963224cd20629", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/workspace/api-contract-review-workflow-eval/openapi.yaml": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/workspace/api-contract-review-workflow-eval/src/routes.ts": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/workspace/api-contract-review-workflow-eval/src/session-routes.ts": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/workspace/api-contract-review-workflow-eval/docs/api-contract.md": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/result.json": "c342198b1658549510559915ea82e0a9a17d86b3eb32da91c80995a5aaef4ecb", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/score.json": "b8b99e9fe7793d9ac4d6bd5902b393a641f31f339829a6f53a6da4c63a383c01", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/metrics.json": "35f8bd229f5a9fe277e6c0b1c0d9bbc39498ac1279a801ae667fcb17298cda44", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/codex-inline-judge.json": "20ea8de02e730a2b13235eb03ac5f87e7d2572388b6a1d6c4efaa22e91bb72bc", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md": "fb46dc40e4f2649bb536c4c8675ce44b5f8f273a851d557e0f679527a6f7c699", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/artifacts/api-contract-review-workflow-eval/tasks/eval/review.md": "b75ca7119b37bc9480106bcd478089319e5b7b4e9635dd53e64dc9050d755c86", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md": "a29d8ccb0ada4093aeb54fdd6888bb72eb9d0c1ee133406e5b244d1bec633fbd", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/workspace/api-contract-review-workflow-eval/openapi.yaml": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/workspace/api-contract-review-workflow-eval/src/routes.ts": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/workspace/api-contract-review-workflow-eval/src/session-routes.ts": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/workspace/api-contract-review-workflow-eval/docs/api-contract.md": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/result.json": "8ff57592e72a75356c6ab1f1d2346f41e6349b2c364c96587c2cf19cd783b0b3", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/score.json": "58fd312323193d10b25d633d21bcc4d0780dd314e02c03ea316896823dbc8938", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/metrics.json": "c8f52be1d22494e4db9d38d373c8ca001ddeea34afb191079e9e86d3406b317a", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/codex-inline-judge.json": "9aca8a979b9ad7ebafa513d2280b8d4a2273abdce74296811b1d0d4e8600c1db", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md": "0909c447295f69b9d74a01dbdb36eb7c4b034073081811afe7d8f057f7124df1", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/artifacts/api-contract-review-workflow-eval/tasks/eval/review.md": "e5cc777709a85552ac504fdb009dabe6a77390c98dc5c78257bcf9f69259fbe5", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md": "36a4585c03a691cd04458a2ac53704241d0c0ede7abbc79a5087dd7e3c079c8c", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/workspace/api-contract-review-workflow-eval/openapi.yaml": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/workspace/api-contract-review-workflow-eval/src/routes.ts": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/workspace/api-contract-review-workflow-eval/src/session-routes.ts": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8", "gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/workspace/api-contract-review-workflow-eval/docs/api-contract.md": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/result.json": "43be5c5d39478f9ff53b2f307c7161b1845e2bc2b2211dd1cda56357092f313e", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/score.json": "c30d6afe6f1e10bec70afcf1ca92d4a9e2de5452745ef58ad6f6f9a29245f7e0", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/metrics.json": "73199d2ad4c3c23984bd65f42d511f53d7b8badd412fd4c88fd232a13a7c4758", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/codex-inline-judge.json": "9df5bc348dc2afe180d65cf3a098e3c26850e3a1ff01d8258a91a66978593b1d", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/workspace/security-maintenance-eval/security-review/REPORT.md": "40d3db3f3726284716508e60d9d88064bdc24cd3f2b9a9d15b6ec620e48a1264", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/workspace/security-maintenance-eval/security-review/findings.json": "41fb00955c9bb609c29bd70be4ea968953e8c2985b315c0fa4a3254f5fafa3a7", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/workspace/security-maintenance-eval/src/admin.ts": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/workspace/security-maintenance-eval/src/profile.ts": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/workspace/security-maintenance-eval/src/reports.ts": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/workspace/security-maintenance-eval/docs/security.md": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/result.json": "77bb6cde8426e8b17a5a001a1dcfdcfbfa4c3cc6ef3c4b5e255598123f185b50", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/score.json": "0d9b2a3672f1e2b7241fa87d9b00422e63393da3349afbde99265091ad032393", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/metrics.json": "8d9bb7feca93765898d0d2e3ac8399764dac59532fabea0478efbacbb0cc7f80", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/codex-inline-judge.json": "b9fb2c4f7c55f67966346ada33fae4668e941342d147587bfaecca173cfa3018", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/workspace/security-maintenance-eval/security-review/REPORT.md": "53207a87fd02a05a9558bfb563f66d9d46411cdbd5cc18f451026b58a9fed967", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/workspace/security-maintenance-eval/security-review/findings.json": "a6522b60268a47f85190b5eb148d8028b4bbe1f72d4df2e89880bcde3035b75a", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/workspace/security-maintenance-eval/src/admin.ts": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/workspace/security-maintenance-eval/src/profile.ts": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/workspace/security-maintenance-eval/src/reports.ts": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/workspace/security-maintenance-eval/docs/security.md": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/result.json": "43d1c35be4102d75a667fa39717bf6fbc057146eef08d79339b08cd37207bd4c", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/score.json": "4bb6ca8eda03d68129784744d84b54eb3808eff9d05d0d78c2649e14e289f8ad", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/metrics.json": "01fc4da9ccb613ce0257bb2ae367406d1004e7a03c3bcb107b93ec6f3b59288e", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/codex-inline-judge.json": "529653676e46443c3ec047c19ff371aa0dea2063c6f64343702e6fea33eb101e", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/workspace/security-maintenance-eval/security-review/REPORT.md": "6c6c0238ba9cb247668122313750efe7758b1b56d1e68b3974da04b5d33fc1d2", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/workspace/security-maintenance-eval/security-review/findings.json": "7ec0e91bf4e7ac73fb1d6925668274872a26283904d629e4fbddbedf76ea1495", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/workspace/security-maintenance-eval/src/admin.ts": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/workspace/security-maintenance-eval/src/profile.ts": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/workspace/security-maintenance-eval/src/reports.ts": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790", "gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/workspace/security-maintenance-eval/docs/security.md": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk/result.json": "35effc2e3cff7083ef852a083bb37c2407a4e26ce3c058c1d38db3649fea4cde", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk/score.json": "ca5c3d25dd896f0749dacc661771dd2f1681e1a3f8ff355c70ddd259d063d161", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk/metrics.json": "cf671c1d6c6afc7f7e851f0032a34e81bace54a16d0c32bd34a81ce99a8c414b", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk/codex-inline-judge.json": "5e2900b178bdc627267a703d09d33ec725508cb2c6014ad28f3974e46e562198", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i/result.json": "2724e260bc3b3da6f0492c45a9c2f7d1df4e40be9d441c47afdbf0f8fa96abca", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i/score.json": "970ba2c5fe9f1f2f220facdeea57f416e8b64e9ab0210f9b6a8d9e6cbfab1d4e", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i/metrics.json": "43995bac659bd286397261a9d99870a735c9981c7fabb5655655ae4fe85ec8d4", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i/codex-inline-judge.json": "85fce5df73d8e2c1e37d1f8c9a7322829365ec7114589457442b4524df23a81c", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/result.json": "16af2c3a73721499978f057e41a6be09a1d360ae642a351a9ebb6489ea45d42c", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/score.json": "b6b271c9250d9d44582be037ba789f2c44bcd39720ef19e9a39512625cb6a0cb", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/metrics.json": "9dd88c4ab9360daa6a2e28e76cddf12f07d196830b492e32d097fc39a459012d", "gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/codex-inline-judge.json": "493573c75c1791740c7ffc8164ce682448f739f040c37f9b90fa5f6f0f2cab24", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/result.json": "342e21434f36904779395371ff11567064455841886c1d4e80197729b00c4780", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/score.json": "558964afbf88d47b7262d6453fb7d199cae3a1c707371df72a635504eace7047", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/metrics.json": "cc7540c0cbbbb02c424131895c50142766a07a8bbe9f876d2c9dd7af8b1f0bf7", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/codex-inline-judge.json": "d034f4eca79b4a4cfb67b9d9c842b871e2fb8b3370f07b5f87b07ca601bf7642", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/artifacts/profile-cards/tasks/1/verification.md": "e0ad9b828fef7ba781794e1028edd24ba3919d387eee6bf02c18984bbfad6990", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/artifacts/profile-cards/tasks/1/sites.md": "86764c72ff70c1c28847619902513b9d291ff8dd5199b4696e08a807a72681ad", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/artifacts/profile-cards/tasks/1/change-notes.md": "dacd6451e1d0cf7898e690468a5b28e22441dafa9570b2524d669b917735724f", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/artifacts/profile-cards/tasks/1/review.md": "e6406c1bbc6e490165a2a2a2beb252ecaeea1ad167bf9068434d522de1a2b39a", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/workspace/profile-cards/source/rename-brief.md": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/workspace/profile-cards/package.json": "d69e9390722b6b5ae20c94d274faf88a16fa95d57293a7d2fcf5ecad97c0650b", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/workspace/profile-cards/src/api.js": "78619f2daba1ddc732dca868c5cdbb606b5c0aad0e8aea977227d361556d5316", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/workspace/profile-cards/src/index.js": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/workspace/profile-cards/src/profile.js": "a723dc6e94beda9791e150daf98b27ae01f94968a7ffecabd4e0ad736641f3b0", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/workspace/profile-cards/tests/profile.test.mjs": "cb6f1aa1b2663a90c2452f43bdadba53df012bb4c7b022366d02545d4ead852c", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/workspace/profile-cards/CHANGELOG.md": "51760c8cae86a3fd0b4ff6a0ea029d890ebc0e64045f32b7bed53e2da145e5ea", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/judge-context-codemod.json": "eac3beffb4f23f2e3dfb11ec741e28307f74f583f9f3b6599f9195a3a2bca458", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/result.json": "74c4112bc432622b8ba3513c6f3d24434e0fdca1ea0f3ea74fae1e531e7eb727", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/score.json": "b55c5b3ab9ed1a06c5b4dbf78a4351c3bb98833708cd51b2823c128328aa9861", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/metrics.json": "e00078132d5740f3eecc9f7512a9cdcba6dbf37f8085cdcef649feb1aa85be9b", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/codex-inline-judge.json": "774494203a45110c9bdc1cc48769d55538291985b1570402f1fda8b00fc36a24", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/artifacts/profile-cards/tasks/1/verification.md": "a6f98095ef9f7a9906c301be72ceb047e1c4b62f8b80314ab79a4caf4eecf111", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/artifacts/profile-cards/tasks/1/sites.md": "b4d774ec4e33752c38a38be3e7fb3bc0c9e5b51fda0c616ac504d4b153393bf3", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/artifacts/profile-cards/tasks/1/change-notes.md": "4bbef997f0bbaf57478338cede891c3d9c8653ac842d99d1458a348501e2bb59", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/artifacts/profile-cards/tasks/1/review.md": "0451c5efe291c5f28b06b7552f2432ef98710b1ba1fb14e1d6947e2af3141fd1", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/workspace/profile-cards/tasks/1/verification.md": "4acb5e4e344dc2d0f8977e99ebc25a772cea26faffe7bf3c73e03321acd5c56f", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/workspace/profile-cards/source/rename-brief.md": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/workspace/profile-cards/package.json": "d69e9390722b6b5ae20c94d274faf88a16fa95d57293a7d2fcf5ecad97c0650b", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/workspace/profile-cards/src/api.js": "01272894ba13b3d0f0b0c2d643a48ddc1388fe666d5f17d268b6a201905a9b7a", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/workspace/profile-cards/src/index.js": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/workspace/profile-cards/src/profile.js": "b7bfec2ac66313b137b25a1411df47ce17f7d1a648119343b585116d01b70819", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/workspace/profile-cards/tests/profile.test.mjs": "fb92b162b53605262a2032ece1962f8c5deb8ac182c75d98c4202db54402c82d", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/workspace/profile-cards/CHANGELOG.md": "51760c8cae86a3fd0b4ff6a0ea029d890ebc0e64045f32b7bed53e2da145e5ea", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/judge-context-codemod.json": "c94e3dc480c0f5844c8c182f1085ee08b00e0ab21bff67cb2d90e506bdcd1eaf", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/result.json": "fa18bb972431b555e8f391b585257e4e6c352cbf387e87bf3d29b171b50d5216", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/score.json": "b8b9f7263e7962a189490a4f3544e814388e681fa7947cb23629b962a7d0cfa2", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/metrics.json": "731a206a99e7fe749015eb536662e9226f3b9db6bf9f8321f74b67c32f01bc5c", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/codex-inline-judge.json": "5549a5df78227fa406f9d18f33de11efe45cf35e1c3b269bcbe3dd1984e5d93a", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/artifacts/profile-cards/tasks/1/verification.md": "3fa0b32232e58ce59f997c5c61582714b67f37d2e78ae2f2da09edc95f7d6a4f", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/artifacts/profile-cards/tasks/1/sites.md": "9f020ecf4d56800c90483568a6bfd2a00dae49a630276006589f4e4899a400cf", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/artifacts/profile-cards/tasks/1/change-notes.md": "21552571c17c862f1b4f69c5f7c1e9ddabf37a2ebfd0aaaf0ee52f38831ec3c3", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/artifacts/profile-cards/tasks/1/review.md": "a82ad2ac078c55febf6c794f50b473a01881266119dd09b5ef83577ea3faa6bb", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/workspace/profile-cards/source/rename-brief.md": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/workspace/profile-cards/package.json": "d69e9390722b6b5ae20c94d274faf88a16fa95d57293a7d2fcf5ecad97c0650b", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/workspace/profile-cards/src/api.js": "bd78e33b9cf74aec788b859016f4a43e161b0173bd06add93ecc140b791c7a0d", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/workspace/profile-cards/src/index.js": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/workspace/profile-cards/src/profile.js": "a723dc6e94beda9791e150daf98b27ae01f94968a7ffecabd4e0ad736641f3b0", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/workspace/profile-cards/tests/profile.test.mjs": "e607dd92c736e8fca735835ae316d8936baecca5ba9a93ab2ec2ff00f42893bc", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/workspace/profile-cards/CHANGELOG.md": "51760c8cae86a3fd0b4ff6a0ea029d890ebc0e64045f32b7bed53e2da145e5ea", "gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/judge-context-codemod.json": "bebe919112f8553ca6dcdb0210f9a08eb39ea1e9c82c069a5ae9b353f1dd2dda", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/result.json": "896ca1f0a0ce811b94119d7a3a469dec31d63d43188a4c8046557bc796afaa34", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/score.json": "33b596d98c9f1535aaafb9a867d18ffa00f30043f6fa552845d41ee27a7d86ae", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/metrics.json": "ec2778c23a451034d025b354a26c9f3415a84e9c8be5b5ca880837861d2ca92a", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/codex-inline-judge.json": "94bb7f8917d23cbdb97851762f0880b24857a37a179f64cb334efbb5d116b562", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/artifacts/crewlog/tasks/1/change-notes.md": "c6474cac677f66f99208ac3078e36c5d7cb4c37baab7e4f1462dc44aa6be705b", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/artifacts/crewlog/tasks/1/baseline.md": "975ffd8358e077d6a16938fb3ea41dc9c33c66b33c6228f6d7ee3e42cb340493", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/artifacts/crewlog/tasks/1/plan.md": "384240904edc604761507fe7c981fb03870574f715b6688739d177287d03c884", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/artifacts/crewlog/tasks/1/verification.md": "b6a1b442b171a8f6267175cd6dfe15e6f4c1158bf25c04f0a8df9a70bed36a99", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/artifacts/crewlog/tasks/1/review.md": "45d7e843abfee1514e8bc82951a3430ce091050d7568db639cfb043c9795908e", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/workspace/crewlog/source/refactor-brief.md": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/workspace/crewlog/package.json": "06c2f0dc9cbe2ed0b953f8e1e9f3fa7a41326d0b7222b4d26aa35fef2de25c1b", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/workspace/crewlog/src/format-name.js": "4779137dd7acff80d97a674b85055f0344eba270160f090dc5798c9867be7b63", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/workspace/crewlog/src/report.js": "0ea85cd02142f8d5474afea2f71ea3f39d72532e2676b84f2aed299b7bbbb791", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/workspace/crewlog/src/normalize-name.js": "d1dddc05eaf390790717346986ee6417f509d0fce84f01c36c95e26665bf8595", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/workspace/crewlog/tests/format.test.mjs": "80c14472bc71dee5f53f76e4d1428c053ee96bdd68afa984bd583578b4b981ce", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/workspace/crewlog/tests/report.test.mjs": "3e3a3ac8ac564c254e4127b74b745be4ada01d81c263b4c514f47c2225b04612", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/workspace/crewlog/tests/characterization.test.mjs": "14373970f0d38c9c9f0b397ae0066f2ccc124d8a95366814640332296e454def", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/judge-context-refactor.json": "5638aaf42cfd77ddf9a5de819a1a4b49ddd2b2cb0d59bb3a4206437eca94595c", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/result.json": "50e4e011f3e28c26643eac29601c8d0258bff3489034d1cacfb37d2969d2beea", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/score.json": "6760dc43c8caa540f4b3d43fe003c9ca7d75fca3aa7d8fd2df0a0815f2a56853", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/metrics.json": "6112d207f8ba6da0aa76252ac9d667ae4f4ab4a760b533d2cb9659d1db2a3bb4", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/codex-inline-judge.json": "c76da1b1d42ba3da51c43cc6dd15685499f3d551e6b71af92930090e651ec678", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/artifacts/crewlog/tasks/1/change-notes.md": "da08288a7f61c58b2e4e01fdb2f148f664bb8b0cd9a3ef75a8bcc44c40813938", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/artifacts/crewlog/tasks/1/baseline.md": "378449015566ebdfa544af0e92c903cee40e69ca62c14faf0fb8658d34e934ce", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/artifacts/crewlog/tasks/1/plan.md": "f68906cdf911fd14f9178df6394b66c2bd1323f29d40c1530e19e5216c1727e6", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/artifacts/crewlog/tasks/1/verification.md": "40fdc9f35219856df131027bbe7eb88e6bea26d234667d385e6bd354b0e28dac", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/artifacts/crewlog/tasks/1/review.md": "cb88aecf12f3ee6ac1516cf4ec596a265a0efb5169773d59d8b9c443fc48e90b", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/workspace/crewlog/source/refactor-brief.md": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/workspace/crewlog/package.json": "06c2f0dc9cbe2ed0b953f8e1e9f3fa7a41326d0b7222b4d26aa35fef2de25c1b", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/workspace/crewlog/src/format-name.js": "4779137dd7acff80d97a674b85055f0344eba270160f090dc5798c9867be7b63", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/workspace/crewlog/src/report.js": "0ea85cd02142f8d5474afea2f71ea3f39d72532e2676b84f2aed299b7bbbb791", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/workspace/crewlog/tests/format.test.mjs": "80c14472bc71dee5f53f76e4d1428c053ee96bdd68afa984bd583578b4b981ce", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/workspace/crewlog/tests/report.test.mjs": "fc6456020a060a454549183e3ea594d18e313bc33e18375969b5fe2a70853351", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/workspace/crewlog/tests/characterization.test.mjs": "32418eed50fe2df7731620914b2a6b5455fbd6b41afc88015bd15d35482ec20b", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/judge-context-refactor.json": "2acc61ebc2c02cca2effbf49932f7f30fc96ee3f5657a1712e5a7e8f8191aeb8", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/result.json": "fcdb3c5b14510a714c6b817bef9c9b14b80b3ffbfae6076a04975971c1c20260", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/score.json": "19b8219df8c13a28ffb9ad7b8c5c848ec3730534419b0cce22c52e763086ca99", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/metrics.json": "85b181781a72d57477550ad748b92b111739f4c8629de4bd36502b0590d8b992", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/codex-inline-judge.json": "c6c48ded50dfd26c6fc20b62a6b6fea90fe3102b4f259f1c576dbf7784aa0691", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/artifacts/crewlog/tasks/1/change-notes.md": "6dab4f1df57ce6e55a194542f33716ca6e4312451c8d7961e0b71050b968e40a", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/artifacts/crewlog/tasks/1/baseline.md": "bf5f5dc9f00e9db1bc62c3c5d323e31987f74145a2d56786b4c02628ddb4331c", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/artifacts/crewlog/tasks/1/plan.md": "a54eae84114c26d3c278edb8bf5b23e2e47f4216314e09dae84cec64eaca04dd", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/artifacts/crewlog/tasks/1/verification.md": "458ddd1dd6109e4a828887c094940d06ca1c86adf4c230be72527656cb75e016", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/artifacts/crewlog/tasks/1/review.md": "ca7cf9f39d0e536914d9e3ac63941079173c2c6efdc9ddc0fcc90269db70b2fb", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/source/refactor-brief.md": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/package.json": "06c2f0dc9cbe2ed0b953f8e1e9f3fa7a41326d0b7222b4d26aa35fef2de25c1b", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/src/format-name.js": "7c0c3dbc8e96c5cbe1db84171e5790d2a298a696b00e7502c51c5c992a19f3ab", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/src/report.js": "142ddde91160542a93cc2a3f1093cb70c30dcff122ef3fee4cc5c32e755ce3e3", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/src/normalize-name.js": "d1dddc05eaf390790717346986ee6417f509d0fce84f01c36c95e26665bf8595", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/tests/format.test.mjs": "80c14472bc71dee5f53f76e4d1428c053ee96bdd68afa984bd583578b4b981ce", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/tests/report.test.mjs": "fc6456020a060a454549183e3ea594d18e313bc33e18375969b5fe2a70853351", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/tests/characterization.test.mjs": "40d664c0f5ecd1380c60f623a4d466bd08d61085c6896bc5f9ab930f76ec5490", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/tests/format-characterization.test.mjs": "e9646d40e6780303b3effc42a854bb825d88a88d2b07db7cc8225d85db8d5866", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/workspace/crewlog/tests/report-characterization.test.mjs": "6764ba19c9ec9dfd51243dccb292c4974f39068f0692332e7052090337c82870", "gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/judge-context-refactor.json": "d7f822ef086306307b34634b739bb19f51b86f32eff3fadee0d988ea91d0d39f", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/result.json": "75f9c781b8ad3433214f338da6379004da970b3fec98aae247fb9cc450a361f0", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/score.json": "5b54e1bc1ff48b56958318af60ab0a56bbe8985b151d356dac6e515ad873aced", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/metrics.json": "7283c7e2cd977dc180115839b5ccb97c813d0f70f25ea47133c31bbe7bf86de4", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/codex-inline-judge.json": "67b89e64f86adef19d4eb964b21c020fd6f7ff2b0aba380c62a097b8d939a04c", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/artifacts/orderly-pricing/tasks/1/fix-notes.md": "5349607fb0160cf65a15242d9187ae23de18beee12aed5c7afe040472240af9a", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/artifacts/orderly-pricing/tasks/1/repro.md": "ef522aa35bbde80e2fd6166adbfe042c4330d3e53e383b04ec2c34460c30f01d", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/artifacts/orderly-pricing/tasks/1/diagnosis.md": "7fac5cfc6cc61efa0d3654e74409e28db9c6a5259d8e119e41288a1549a99a75", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/artifacts/orderly-pricing/tasks/1/validation.md": "5d655aa2658e7953d5e5dc72d3196d4e889df3c96b28182b42d7d9d34acad529", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/artifacts/orderly-pricing/tasks/1/review.md": "210705628954d768888d546a33b2d1f549e9cefaca40fbdde031cd14a5b59056", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/artifacts/orderly-pricing/tasks/1/blocker.md": "fed5137ce74fa024be69bf9b8b406babe1fe747cb3495d9d74309a0c42a35d02", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/workspace/orderly-pricing/source/bug-report.md": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/workspace/orderly-pricing/package.json": "0bdb539d11fe5cd1b21c74c3f5e3051d57759f9d63c667d6f4f59bc8ab2f67a7", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/workspace/orderly-pricing/src/pricing.js": "4c03c76d7d58077171a545c26ae25b579dbfede2e12348247be0194d54942126", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/workspace/orderly-pricing/tests/pricing.test.mjs": "046f652bdfde245458ac63c1c3b2e08723a2b656772e1f2b4572da57969d4628", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/workspace/orderly-pricing/tests/repro.test.mjs": "3d4bc8b793b4f12edf374bdd1f7bf2a1c15d986a79624079e70798b9e8f4418b", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/judge-context-bugfix.json": "520ad665099f5be13018f1b9d468405df5cc5fe86721a2b45537c4764ed4a491", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/result.json": "6adbecaba67fefb9da0c2b93e6cc54a04e4dfaf958330f590aed9eddbc476af4", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/score.json": "c10254d29e7703adc0af43b566816df4f9fa32245451a4a618ff762de411905b", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/metrics.json": "71845eabdf3b8342266b02a312acc97d456faeb6638f5267fc643f49ae5fbadb", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/codex-inline-judge.json": "a00030b6f48d2e7c14b5d5885a3672b616e82c1bb78b040f864034130154fac8", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/artifacts/orderly-pricing/tasks/1/fix-notes.md": "31b1aa8bcfa74dc172aa59bfd183e876232e33c4cda7ec28f680bc3465158087", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/artifacts/orderly-pricing/tasks/1/repro.md": "796e6413cfdc540452a7b90a6eba4267d9c0d4898b0340bae1ab9aa500c0192f", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/artifacts/orderly-pricing/tasks/1/diagnosis.md": "1a5ac2607119f438f1ce1c151046bcb6e4240c3c7c6efbe6a5dcb2e4a2ebe021", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/artifacts/orderly-pricing/tasks/1/validation.md": "581a4e9ddcacd0f92b5f07680e1188363a39766cbc8d3f9b026405d10ecaa547", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/artifacts/orderly-pricing/tasks/1/review.md": "16d71020be8050b14725df3c0bb5a460935790360d3ec63e02b1eac2530c7495", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/artifacts/orderly-pricing/tasks/1/notes.md": "22d443ee8a9074279ec308cd11081fbbef296c648c7912bb7e2705ed82df8d8c", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/artifacts/orderly-pricing/tasks/1/task-notes.md": "5c7ca230bf2750493fedf762a28a6f7010859924e02dd6986f60e2dcf1aec5db", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/workspace/orderly-pricing/source/bug-report.md": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/workspace/orderly-pricing/package.json": "0bdb539d11fe5cd1b21c74c3f5e3051d57759f9d63c667d6f4f59bc8ab2f67a7", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/workspace/orderly-pricing/src/pricing.js": "4c03c76d7d58077171a545c26ae25b579dbfede2e12348247be0194d54942126", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/workspace/orderly-pricing/tests/pricing.test.mjs": "1e7c2acd8303ebea677b1d1780bcf21ee0c2098c29afae354f7f137f36721c7f", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/workspace/orderly-pricing/tasks/1/review.md": "ca3b2ea3d5963a1e0b923a7abfc2f97db1bb8fdf5007dfc4c297bf9806da28dd", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/workspace/orderly-pricing/task-notes.md": "38edb3251c1a69b02be15bee03f852c7fb30ce383cd9dc387074384a91643673", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/judge-context-bugfix.json": "285b7c80b5ec84594dff1719a2430c6773dd1853d740b1cd9e4ba66f45fa40c4", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/result.json": "56a0c98d5502380584eb5958730bc736f07b4bf36f6cbd122cdf01e1e73df2c9", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/score.json": "a422f467cbfc45f6d69f6754889617cfd5f88e0dbf79de36e082eb0cccd013cf", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/metrics.json": "068b11a9e933f341714d4bacc6fd0e983435327f207909038d8a06b6d9df0b7d", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/codex-inline-judge.json": "ec349247515c82cb1c040c545282f89ba2e39757183d474a49eecc9ff1f6570b", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/artifacts/orderly-pricing/tasks/1/fix-notes.md": "560d215960b60275020b0090ab11cf6a8922204daf444802071402435393eb9c", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/artifacts/orderly-pricing/tasks/1/repro.md": "370754b5cd8083b33bdf10e4708e4ab4a34db5fc9058e6ea1897463d783a3968", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/artifacts/orderly-pricing/tasks/1/diagnosis.md": "05ff488724368de52b75ef54d9369939cb0194106ee746925cff1b12fdd5e661", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/artifacts/orderly-pricing/tasks/1/validation.md": "ae5521a8c89f0ff695ddd167300a8a125465805a20e8badc4c92d42e488af037", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/artifacts/orderly-pricing/tasks/1/review.md": "528364fd240de005a23627c792cc6cfb8e15fc4c353954121fe686b49dec79f7", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/artifacts/orderly-pricing/tasks/1/task-notes.md": "436e7a070d9de7b168641edb8cc4f544c8b17b7bf091ce6ac6705f75d94e616d", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/workspace/orderly-pricing/source/bug-report.md": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/workspace/orderly-pricing/package.json": "0bdb539d11fe5cd1b21c74c3f5e3051d57759f9d63c667d6f4f59bc8ab2f67a7", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/workspace/orderly-pricing/src/pricing.js": "fad23b0c7929def833f4dcb9c3557a77ccb3f3ad92b3be97e462fea4e0a56921", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/workspace/orderly-pricing/tests/pricing.test.mjs": "e0fdd0f6a038c279dbfcf2504cff15728b35d51e4c22645fc6dd6aedf3f6788f", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/workspace/orderly-pricing/tests/repro.test.mjs": "ebbaf63446c01b18a8b70777f18ac0c585f86d3d30d455b6b98a442781e94e87", "gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/judge-context-bugfix.json": "b98a8e6234721bb4ae9dd4b7ba0f0bf8a67e9d26f003b2bbe3f860eb800a1f25", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248/result.json": "f4c5e510ad88f00a82f8c0c847bbd411532750eb0172afd03f524fa1ecbbc377", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248/score.json": "4177cf94247be8c297ff7a8e589bb923aef0e5e975638125ad700f71ab6a305b", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248/metrics.json": "afa67bd281073e3b3360bd533de5df44850feaa517a365dd47635438a6a7e778", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248/codex-inline-judge.json": "7e2eed02eb65e2538c6ac5995754239355dadffdf99312394d4512f7364d58bf", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt/result.json": "338cdc3a57ad0006bea621f8d9784e3e90d553bd112a03663d1f74d4265f788b", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt/score.json": "5d9b1ef20d422c95d230311ff0a1c6e8de4e78053da91f0e5dd77a7a49d900db", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt/metrics.json": "be4bf01fed5944eea184f3f240d3196e2a6c8c25d3445081e6664027ae73f64e", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt/codex-inline-judge.json": "fb78cc1d91c10434b41388583c7f1e36dceb0c9e41b647f47b8c661d9bf0dda1", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/result.json": "0b73d508005d85f3fb0c9e833de8a5753138574ce15096b15c4f43821b9a6dfc", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/score.json": "1e2e0074e503765ecd1ddd17d809fee1a02a062b98c461c412da6a0a112821df", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/metrics.json": "d3866137979cc3b8c152982be5dedbb804d513f853e04cc5dbb6cc4b07cd19fd", "gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/codex-inline-judge.json": "58c1c89fc117c51d0f1cf590d78352a908cc2f59583586c5b4cc469f13ade13a", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i/result.json": "75f5fd89f5f6143022b46d0c9c9af101a2c3fd8b1c691270648f9421cfa6b9d1", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i/score.json": "458935aee0e2dd53c54bf83be1ffadf6d3b02d7440223b8ebe3df5a29a7cb3c0", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i/metrics.json": "78fa2fbd482d8cb9b2fa5ee7d43c40a1d073d082fb4b76f88ed67709bdbabcca", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i/codex-inline-judge.json": "edc43bd4dbb9c7edfe7c0885d8e71c322617296e199d50928d10a8eb9cc1b606", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig/result.json": "4623fdbf81f3800171c312459ea92d1d526802aa4065079c0b854e6a46b93f95", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig/score.json": "dff63797cdc3d81d23b937de11e5e0423d0f31191da9986f4e8cc3b775ab4472", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig/metrics.json": "38bb4b0a2183da41db169a36cadca7f2fe0958fda825e0a1b58a6b7963aac1db", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig/codex-inline-judge.json": "a4ab32652036ba3c841f666caadfed13d52091c80f472960a19830594d61a94b", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/result.json": "ae8494aa9efa5e5da947df0032cacceeb9e71e530ed2ce35994dc0e58df511a7", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/score.json": "4595b7b63b89d6b4584653422425e37c009bf1066bf514fe596a6da48479f13c", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/metrics.json": "7428aff91671dcead1a9008b53c5a2227ea9714f0a6b25834222703580c6b0e8", "gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/codex-inline-judge.json": "bc87665c0bb98c623d5bda8bc4538c9aa0eb420b46641c487547b28d05483cad", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/result.json": "c16276c68d0fd5d897e9d8c8521cc79084474961bd546c5274cbde0b0ce0cc4a", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/score.json": "9100a374ba4ed5c4c1e171cfee9323b7769eb0afef44eaae9d8b33321efac033", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/metrics.json": "46e5e1039305bada0f87b8279d5c40bf2e80a1ac2f85412f9e1adcca1a8c1951", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/codex-inline-judge.json": "34e0716ef26f460cff9f5162c3c46b2657a3ddd6867a44f9398f3010f713f06c", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/workspace/fieldnote-studio-office/report.md": "e1c3045995bbad0ae0175843b4a86b28d0539e1ccbdcd1cd23523a0ecd081c98", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/workspace/fieldnote-studio-office/clients.json": "1aa5d375dd5ecaaf2470f8a402b6b57c8a1c53508f0b49dccaf9baebf8d63170", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/workspace/fieldnote-studio-office/ledger.json": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/workspace/fieldnote-studio-office/invoices/2026-042.html": "00fe29850ba09322bdeecd8087ca78367638f498101d98e01060f15ffc4bff3b", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/workspace/fieldnote-studio-office/invoices/2026-043.html": "2eca4354805c9fdce3c603096e7649a4ba553163b38031cfdb8c8912f6f7a4a0", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/workspace/fieldnote-studio-office/invoices/2026-044.html": "5e248663bb6904a9008c3820be87d746d9436a19a228aba7ca86fdd2727445ee", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/artifacts/fieldnote-studio-office/tasks/1/scope.md": "ca89f0d717bd93d22cc55c0ece845f58b606239724783c4c867e782e6a7d45e1", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/artifacts/fieldnote-studio-office/tasks/1/billables.json": "125dccb2a75bd46a3f9bb54ebff1ac2cfbe5f5021326ad332fc00474915f5a2b", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/artifacts/fieldnote-studio-office/tasks/1/draft.md": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/artifacts/fieldnote-studio-office/tasks/1/verdict.md": "1a22dcf9240b795c8f77eb3691b1d4b3d792750d1781e97efb18a64c450b148f", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/judge-context-invoice.json": "7f1cc39e48ea73100cb4a21d8801474f0117112b2b9eda5cb0f20f3160f395c4", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/result.json": "9283e29416203293afcc5bd430e643488e0872220515f893c2566f1acbfab9d6", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/score.json": "3ae0969693719dcbfa1ed7cf42c61cb4fcad5844e0d0f1f9e92ef09644a9611c", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/metrics.json": "9e86bcba12efa049f4d0b7a037c640e5afb5bae1c774af2a16cf15d1a35e8f74", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/codex-inline-judge.json": "a28ac1b3aa02b66cb24a4a4ee9f07daee23965f9c8f583ad4c2b609131f39987", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/workspace/fieldnote-studio-office/report.md": "b0ff0e73b7f1a3004e33aa284dcf11c62707e07d9ecb02f7252cf52da2f3508a", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/workspace/fieldnote-studio-office/clients.json": "1aa5d375dd5ecaaf2470f8a402b6b57c8a1c53508f0b49dccaf9baebf8d63170", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/workspace/fieldnote-studio-office/ledger.json": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/workspace/fieldnote-studio-office/invoices/2026-042.html": "196a8aa6886980f7753454d4c27f6355925ed4641c6d6067305c43de3f27d063", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/workspace/fieldnote-studio-office/invoices/2026-043.html": "13724d1fb807d3b1b6bac439dd46b89debccc4b41567a14fac94120b1242fa73", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/workspace/fieldnote-studio-office/invoices/2026-044.html": "c017136dc568c187533a5c285545247b99140aa19e638b7af74220cf777d64c0", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/artifacts/fieldnote-studio-office/tasks/1/scope.md": "6d99866fbd63cb481338d12d7dd7a6868051fa4e7aa442929fdc17efe1ffc9b3", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/artifacts/fieldnote-studio-office/tasks/1/billables.json": "125dccb2a75bd46a3f9bb54ebff1ac2cfbe5f5021326ad332fc00474915f5a2b", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/artifacts/fieldnote-studio-office/tasks/1/draft.md": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/artifacts/fieldnote-studio-office/tasks/1/verdict.md": "b8bd5342e47768781831bc21c4ffd35e2422487ff85529e2f78f3aa38c36b09a", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/judge-context-invoice.json": "0ee9d81eee6735f49158346317540c55ff1d42559de4fb8e374303b4b2496ca5", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/result.json": "e0567a9058c1b2ec98e0dbc2e293077b018240df378049144e21e95316b20e41", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/score.json": "85a32b2938dff4e8cf1b32e1d42e390d555bf36c669196e926a2f764cde94d67", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/metrics.json": "80ad2e9ee3aa2f6994f30943fcb1efce57b4c21a6c63e51359a1f5cce5e15e9d", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/codex-inline-judge.json": "3a23eceeeb30648a102f212b7788c0d7b544532d29809cc02cee0364a765d21b", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/workspace/fieldnote-studio-office/report.md": "26e309608ab8421941c90db3e41173e3d78b265971d5a46a68c1aaded60dc68b", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/workspace/fieldnote-studio-office/clients.json": "1aa5d375dd5ecaaf2470f8a402b6b57c8a1c53508f0b49dccaf9baebf8d63170", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/workspace/fieldnote-studio-office/ledger.json": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/workspace/fieldnote-studio-office/invoices/2026-042.html": "65e693b1b537fa5c4560c9c09d27290f03d3a79c7ec6d24039292fcf744f4047", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/workspace/fieldnote-studio-office/invoices/2026-043.html": "04e66cae823071016b237491c06fe579805031aba24a1339b7d8472c39b0a3ce", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/workspace/fieldnote-studio-office/invoices/2026-044.html": "7834b0bb6b02475adb057fa3760376ca68e6da2d03d04849a27f5bd02de68078", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/artifacts/fieldnote-studio-office/tasks/1/scope.md": "ca89f0d717bd93d22cc55c0ece845f58b606239724783c4c867e782e6a7d45e1", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/artifacts/fieldnote-studio-office/tasks/1/billables.json": "125dccb2a75bd46a3f9bb54ebff1ac2cfbe5f5021326ad332fc00474915f5a2b", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/artifacts/fieldnote-studio-office/tasks/1/draft.md": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/artifacts/fieldnote-studio-office/tasks/1/verdict.md": "58b1d02de5f60a98b9f7bceb26990cf9c604cb7a8a16df0dd367bfde712656b7", "gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/judge-context-invoice.json": "bcd8a0a6b271aa620b616be0aaca4a44128292ad1699c361ec3d5d1b0b2d3df2", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb/result.json": "7215d7571075b176947685f233bd280ed0184cf13e668cb46d49997d56b7bfc7", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb/score.json": "efa63d89f3c30b7e58b5f3b06ff11db2817d53e1ee2e7a6f0aa84b3a719410e2", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb/metrics.json": "ae387c116ef5abd9aa34d8e4111001af0b4f7a64c6dcece347b8e5163fba0ec1", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb/codex-inline-judge.json": "1cf9a1a72b4e9372b5078946e3b73436fe5e4b651cc366d3d4c4262c7ba24888", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s/result.json": "3a346068df0a008d32e17d639c95fb2cd93a61a3a7e41a2168a98acf50da246c", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s/score.json": "89bd545d14d56ba16e963100bb4d7c2ce2ee56f72baf32bb112f95c6d8d7d34a", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s/metrics.json": "efe21ddf0625fbf74cd62b627cb10651bc39b5854118bb9cd86cbe4b32e1ddba", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s/codex-inline-judge.json": "c8b5f2152e587fd5abd3790f9c35083812b8a99655618ee44406d90c47a54277", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/result.json": "279048432fe402de35b437756e25a778d4f4604d53529cb19ac6a6fb8d331231", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/score.json": "02d8ad43ceff2fd7bd786212c653ea3f46be919bd9df108026e64b2e4c6a25e7", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/metrics.json": "fbef90f46e8222b3d3ef6314ee434c1399091fb47586e00a9e090224021b0408", "gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/codex-inline-judge.json": "307111b49f3877291a7ff5aade176548e15487e3bfd27dbef80594611daa4e59", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj/result.json": "52f2f9ce4bd323909c29cd818dc740c887c6801bef69b7a49113b853e76de4ed", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj/score.json": "a5db8b3771993acd0751e869b601bc9739a982ecb71c3e0becec8b728f7569c9", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj/metrics.json": "ac394f1e55b84e8c1210810b1effb261ba380dd75010c173a5f168eb6d21e468", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj/codex-inline-judge.json": "38e3711f2341b2d3a4da27288db5c65525c2d67b73ba1e3026072aa70357f93b", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl/result.json": "3238e4f7304f01defe83709b5e3300cdf24d27e378280f48bec174d2563aab53", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl/score.json": "d713e45e7ec677622a7534fc401a405b4e64e3bbd4642bdd08e4930eb00d77f2", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl/metrics.json": "b756431bdbf6b506f9e05065e3986c52982af29c5b06cfe4b54b3d8527ba8001", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl/codex-inline-judge.json": "71b3580e50aa06d576c3923386ab7529f306772dfee0edce7eebba63700c279a", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/result.json": "84e04fcfa4bf74e44f5c02ef60879cce65e75648cd3864fbf2d105c61113eef0", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/score.json": "98e20965322b59e91967fc42b5119d3e9bf26ea5405b5eeec11df7469993ca30", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/metrics.json": "90e4eecc1777fefae7665f59cf9dc47f87cefc932f01e96698e3ffd96fb1d066", "gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/codex-inline-judge.json": "40504f2ed57258aa777d0b549cf98e8602f1fee25832ceefb25f8336620a193f", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi/result.json": "a396e3edfc3dff27ad29a246855341afe5d7c08ab1074f52b1d8169aab58b080", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi/score.json": "182ba35034c5c083e317a3b6c74aa6024720b759cb7bbae56790d4c5007778f8", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi/metrics.json": "c0c9574554f18a32d865ccfdd38a9dbed71bd1457d1957e04110d176392e5892", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi/codex-inline-judge.json": "073c077f1a7eae63a44d7f65eca7beaa492bbe8077ac77718d0a79ac8154593f", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n/result.json": "72f2c074e36b758f792f1f04e2cb8a891f682cf52d8cd40538a88fd4c09a30d2", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n/score.json": "bf146e2e6532bff65650e5f3c28b8f5a60e4d869bd38e705e3a48444f6aa1f1e", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n/metrics.json": "1e1848f80c290ce3251c287a7d64841881ede764a120d6aaf4215905f1718fb5", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n/codex-inline-judge.json": "3c6e2fd16683cfdca114ada4be6ccd441b5dee0ef77f9972b291187fb6214fde", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/result.json": "16326255fa073d52ebc1a200798a19fdfb653bced7fbe7b92e57dfe6f50f7588", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/score.json": "43d615493e2f6a86e96246b7cb8eda9db82babebab6156d25c5f9f79b4e090c6", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/metrics.json": "08a1c9008ef12bec8ccd768d9e34b7382017efd9c321aa04dd6f66dc38f4e380", "gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/codex-inline-judge.json": "e5469da662ffa054702f9e35ad0c05fddfecf98c11bf3b065d741a14bbe93db4", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7/result.json": "5bab1cca45e42cd1a45b0663e40b204545de15bde4f8d3bc3296c45abbaf9de4", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7/score.json": "abb6c55e768de0c080b83164712f2c4b87f430278e83ea0a5f6b882dbdbfdb92", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7/metrics.json": "32327f51042e2f85318f15d34ebcf6f466f9c6f163ecfe578d9ba70b608f2e0c", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7/codex-inline-judge.json": "177347cb3592a45b0079288188b2aee29c728e9e44cc5fa5e7fbb3b45a322f22", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr/result.json": "2f24a7535d1650de0b549cceac31838f935a32d1c80a6d512999bb4d552d31c7", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr/score.json": "b4b7f039ae5db9b016abb49a7a63e2317008c660e40e16968ed8d629066f5a9f", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr/metrics.json": "9da47565854dedc3ce25b1d981f56a72cf0f9e3d7b516e5ac1c34a9b95fa89df", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr/codex-inline-judge.json": "01cd301567de3efaa28c9c49a1052a0b01bde9b5184ae45579f1ebb648b5f86e", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/result.json": "4e7a96d3110c3cd802c35fa50878d49a65974defea8c6372c9d1cae3dc0a14c9", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/score.json": "e45fa7b5d327c4606d475263c49106c3655945525f25683748b53b15b0011b40", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/metrics.json": "5ad4da215c9eb2aad42faec8d644234c201db686cc0c2f610946dfeda62a9600", "gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/codex-inline-judge.json": "6830e4f92ecb0df94c90db36af1b89fb0b34936c06ff2902559d3e32f9580800", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p/result.json": "bc3a7fd516a8ea8f4526e47adc37dd710487c3dfc1af1192afcd60cf0dc49953", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p/score.json": "b8dc7f30661862be90b44927c4adb68727f4c2f89da548603cbd7da5fd0ea11a", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p/metrics.json": "a6033e933b929946796af32fef5eeb9c53c8a42db26a33c2eeb8dbfc40181656", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p/codex-inline-judge.json": "9e6d424c25f7155922a816ce2da43f57db1cfde884d7e8a9e637f673446d5f1d", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f/result.json": "6653487baa8f538739811eb4e08e4656a0840ab10c09ddfc42d49cfd3cf1e171", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f/score.json": "efb6d5c7ce5486649ba66d5811cabefa927d297731659d21d991b35717e79039", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f/metrics.json": "8d83a9049d378fb388680a6dff31673aea507f21dc5fb53ef9c662a2a874e24e", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f/codex-inline-judge.json": "249e7e947379fc52b48fa32555536ba6dac530fd6fa8cb926734b70440faa7a4", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/result.json": "5384e48c17010d2211c93aad3e12e43c00dd7e1f260958c5a07abb7e5d3a7c00", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/score.json": "26e31da51427f13b9a659d26e3893c69198282ce10d3aad08732275a5a8311c3", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/metrics.json": "9e98d1480b0be97e800862e7ed527eec3b300fc2b51167e460c63db37a2aadc9", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/codex-inline-judge.json": "5c45ccd2de64d52a15fe3ca311bc013fac58083e4320a6b4f0ad5adbc2e32b14", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/workspace/inventory-health-check/index.html": "17ecc31c979781db38121622cde13f8f69ebb46b256cd5bae872a6c5a8cf5418", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/workspace/inventory-health-check/out/inventory-report.json": "285eeba48052b57593bd41c0fec980caf1afbb7e647cd4d7bf3314bc1d955084", "gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/workspace/inventory-health-check/data/inventory.json": "2a6f44cb0bb8d193c2abf527264e828fbf882e8a744180a786e784fe0688b439", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/result.json": "316ae744d16e0ad2179ce1487cb69d51d75e75c07d29de544a2a6c752250368c", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/score.json": "5428e1e65b3ce8aed11755ec6c94da719c7ba4cb324922b01fddade4266aaa1e", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/metrics.json": "823002be847c747e7e718ca171a469d774bb5742b8047fc6ece33bcc3db64aa2", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/codex-inline-judge.json": "1d9ab3569d2488122eab76a5ad198299debdfcbafd57f5a74d139ba3987473a9", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/workspace/store-health-sweep/index.html": "efdddbca93c438ccaa0b9317d280b4ffbc8c8c1a338b89e68cf98e1571e4b9e8", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/workspace/store-health-sweep/data/stores.json": "8bf728bad631198134c68817a9d08eaf08804c45fe27af05c1d8433c7e573e22", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/workspace/store-health-sweep/sweep.py": "d598e70af566ff3ffd2a9127755818555be869809f49a82e214cdf877d289194", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/workspace/store-health-sweep/out/store-1-health.md": "724d1399625b8769f7a20dc266d34e140b4d7b5c32bde433d31858a6eaec4254", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/artifacts/store-health-sweep/out/harbor-health.md": "8b3d2c81b6c85386a3c222e348350ee00b9ffd3426c0fc18339ec722d61f8236", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c/result.json": "b3b310334889f5c909860236c2c41f4d9d4ade4f92701e8870adbec5f13f6c00", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c/score.json": "389836d285afc421377473bb2124bcc8f5c257a555cd85bd1fbe7869260950e6", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c/metrics.json": "127cc296de94249e240fc7340a6b2828615378878adf2f4356d6b174cc3c9657", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c/codex-inline-judge.json": "ad0ac0b15f2071f0639c34c48af3fd8cf68d88661c79ba3d0582459a1b2703eb", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556/result.json": "25d15601798d72efeff6ebda4608f31f6f969181542e8bfeafb701350fe14658", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556/score.json": "caad74e530ee12f35e33dfeaf85109355f7be6f3317f28d224193ac37a66228e", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556/metrics.json": "67a81c5e517847ee768690e5df7f1434b5e5fdc1652550e12b688d7daa5d7773", "gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556/codex-inline-judge.json": "e40984d8ea7c00873203d1fea3703c331cb5c3dc867becfb32040ae38d5a2439" } } }