{ "schemaVersion": 1, "publicationId": "2026-11-03-windows-saluki-native-0-1-57", "status": "published-reviewed", "counts": { "planned": 129, "attempted": 128, "runnerTerminal": 126, "eligible": 125, "runnerExcluded": 1, "interruptedUnscored": 2, "unattempted": 1, "entries": 43, "distinctScenarios": 41, "entriesWithThreeEligible": 41, "passes": 98, "eligibleFailures": 27, "publishedNonModelFailures": 3, "sonnetReviews": 128, "sonnetStatusCounts": { "scored": 52, "notApplicable": 59, "noArtifact": 17 }, "originalCodexReviews": 128 }, "combinedDescriptiveSummary": { "recorded": 126, "eligible": 125, "passes": 98, "failures": 27, "exclusions": 1, "passRate": 78.4, "meanComposite": 8.091999999999997, "axes": { "completion": 8.224, "quality": 8.472, "efficiency": 6.1, "behavior": 9.72 } }, "suites": { "core": { "combined": { "recorded": 33, "eligible": 33, "passes": 30, "failures": 3, "exclusions": 0, "passRate": 90.9090909090909, "meanComposite": 8.845454545454544, "axes": { "completion": 9.181818181818182, "quality": 8.98181818181818, "efficiency": 7.803030303030303, "behavior": 9.090909090909092 } }, "original": { "recorded": 22, "eligible": 22, "passes": 19, "failures": 3, "exclusions": 0, "passRate": 86.36363636363636, "meanComposite": 8.518181818181818, "axes": { "completion": 8.772727272727273, "quality": 8.472727272727273, "efficiency": 7.613636363636363, "behavior": 9.090909090909092 } }, "continuation": { "recorded": 11, "eligible": 11, "passes": 11, "failures": 0, "exclusions": 0, "passRate": 100, "meanComposite": 9.5, "axes": { "completion": 10, "quality": 10, "efficiency": 8.181818181818182, "behavior": 9.090909090909092 } } }, "productivity": { "combined": { "recorded": 39, "eligible": 39, "passes": 28, "failures": 11, "exclusions": 0, "passRate": 71.7948717948718, "meanComposite": 7.969230769230769, "axes": { "completion": 7.794871794871795, "quality": 8.505128205128205, "efficiency": 6.089743589743589, "behavior": 10 } }, "original": { "recorded": 26, "eligible": 26, "passes": 20, "failures": 6, "exclusions": 0, "passRate": 76.92307692307692, "meanComposite": 8.176923076923076, "axes": { "completion": 8.153846153846153, "quality": 8.488461538461538, "efficiency": 6.4423076923076925, "behavior": 10 } }, "continuation": { "recorded": 13, "eligible": 13, "passes": 8, "failures": 5, "exclusions": 0, "passRate": 61.53846153846154, "meanComposite": 7.553846153846154, "axes": { "completion": 7.076923076923077, "quality": 8.538461538461538, "efficiency": 5.384615384615385, "behavior": 10 } } }, "developer": { "combined": { "recorded": 30, "eligible": 30, "passes": 22, "failures": 8, "exclusions": 0, "passRate": 73.33333333333333, "meanComposite": 7.816666666666666, "axes": { "completion": 7.733333333333333, "quality": 7.930000000000001, "efficiency": 6.166666666666667, "behavior": 10 } }, "original": { "recorded": 20, "eligible": 20, "passes": 19, "failures": 1, "exclusions": 0, "passRate": 95, "meanComposite": 8.844999999999999, "axes": { "completion": 9.65, "quality": 9.32, "efficiency": 5.75, "behavior": 10 } }, "continuation": { "recorded": 10, "eligible": 10, "passes": 3, "failures": 7, "exclusions": 0, "passRate": 30, "meanComposite": 5.76, "axes": { "completion": 3.9, "quality": 5.15, "efficiency": 7, "behavior": 10 } } }, "complex-work": { "combined": { "recorded": 24, "eligible": 23, "passes": 18, "failures": 5, "exclusions": 1, "passRate": 78.26086956521739, "meanComposite": 7.5782608695652165, "axes": { "completion": 8.217391304347826, "quality": 8.391304347826088, "efficiency": 3.5869565217391304, "behavior": 9.782608695652174 } }, "original": { "recorded": 17, "eligible": 16, "passes": 13, "failures": 3, "exclusions": 1, "passRate": 81.25, "meanComposite": 7.687499999999999, "axes": { "completion": 8.5, "quality": 8.6875, "efficiency": 3.28125, "behavior": 9.6875 } }, "continuation": { "recorded": 7, "eligible": 7, "passes": 5, "failures": 2, "exclusions": 0, "passRate": 71.42857142857143, "meanComposite": 7.328571428571428, "axes": { "completion": 7.571428571428571, "quality": 7.714285714285714, "efficiency": 4.285714285714286, "behavior": 10 } } } }, "performance": { "all": { "trials": 126, "decodeMedian": 8.35, "decodeMin": 2.57, "decodeMax": 9.41, "prefillMedian": 369.46500000000003, "prefillMin": 118.45, "prefillMax": 644.34, "peakGpuMiB": 10654, "peakRssMiB": 17260, "peakSystemRamMiB": 31721, "summedTrialHours": 88.64472888888889, "medianTrialMinutes": 27.778950000000002 }, "original": { "trials": 85, "decodeMedian": 8.46, "decodeMin": 2.58, "decodeMax": 9.41, "prefillMedian": 381.48, "prefillMin": 118.81, "prefillMax": 644.34, "peakGpuMiB": 10654, "peakRssMiB": 16959, "peakSystemRamMiB": 31721, "summedTrialHours": 62.68504055555555, "medianTrialMinutes": 27.470766666666666 }, "continuation": { "trials": 41, "decodeMedian": 8.17, "decodeMin": 2.57, "decodeMax": 9.25, "prefillMedian": 327.68, "prefillMin": 118.45, "prefillMax": 591.5, "peakGpuMiB": 10652, "peakRssMiB": 17260, "peakSystemRamMiB": 31506, "summedTrialHours": 25.959688333333332, "medianTrialMinutes": 28.30025 }, "round3Early": { "trials": 16, "decodeMedian": 8.585, "decodeMin": 8.17, "decodeMax": 9.25, "prefillMedian": 450.44, "prefillMin": 346.57, "prefillMax": 591.5, "peakGpuMiB": 10652, "peakRssMiB": 17260, "peakSystemRamMiB": 31506, "summedTrialHours": 5.1771575, "medianTrialMinutes": 6.6317916666666665 }, "round3Slow": { "trials": 23, "decodeMedian": 2.79, "decodeMin": 2.57, "decodeMax": 8.4, "prefillMedian": 157.7, "prefillMin": 118.45, "prefillMax": 380.4, "peakGpuMiB": 10646, "peakRssMiB": 15786, "peakSystemRamMiB": 31419, "summedTrialHours": 16.6970975, "medianTrialMinutes": 49.31738333333333 }, "round3LastTwo": { "trials": 2, "decodeMedian": 8.39, "decodeMin": 8.36, "decodeMax": 8.42, "prefillMedian": 341.06, "prefillMin": 327.68, "prefillMax": 354.44, "peakGpuMiB": 10648, "peakRssMiB": 14413, "peakSystemRamMiB": 30903, "summedTrialHours": 4.0854333333333335, "medianTrialMinutes": 122.56299999999999 } }, "hardwareFit": { "gpuMemoryMiB": 12227, "systemRamMiB": 31865, "contextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "fullGpuResidencyDemonstrated": false, "sharedGpuMemoryMeasured": false, "pagingMeasured": false, "recoveredNativeEndpointEpisodes": 8, "explicitCudaOomObserved": false, "explicitContextOverflowObserved": false, "caveat": "Memory values are sampled peaks from different trials, not additive; GPU peak includes image-generation work. CPU offload is intentional. No inferred cause for throughput drops or interruptions." }, "conditions": [ { "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "environmentHash": "688bb13249f9e1905a35004e7b522f2c9b7ed05aa404a535be89f6069d61d44d", "definition": { "version": 1, "count": 3, "suites": [ "core", "productivity", "developer", "complex-work" ], "generalist": "auto", "cliAuth": "login", "targets": [ { "id": "saluki-cuda", "provider": "llama-cpp", "model": "underdog-saluki-27b-iq2-mix" } ], "sourceHome": "\\.gezel-dev", "scenarios": [ { "suite": "core", "id": "tictactoe", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "petshop", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": "sdxl-lightning-4step", "judge": null }, { "suite": "core", "id": "tankcombat", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "schema-migration", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "failing-tests-spec", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [ "vitest" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "symptom-debug", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "data-wrangle", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "incident-postmortem", "timeoutMs": 2400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "ops-runbook-anomaly", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "plan-and-estimate", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "conflict-synthesis", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "constrained-comms", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "records-intake", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "plan-and-estimate", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "meeting-followup", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "meeting-brief.md", "artifactKind": "markdown", "contextNote": "The August 27 transcript is authoritative; old-agenda.md contains rejected proposals that must not become decisions.", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ] } }, { "suite": "productivity", "id": "craftbook-spreadsheet-model", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "scope.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "productivity", "id": "conflict-synthesis", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "docblocks-theme-roundtrip", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "theme-report.md", "artifactKind": "markdown", "contextNote": "Theme inference returned exactly these values: theme id custom-brand-reference-pptx; heading font Kelder Grotesk; body font Marne Text; primary #2E5E4E; background #F4F1EA. The slide count and comparison figures depend on the session's own conversion, and a deterministic gate already checks them against the saved deck and the comparison result. Judge what it cannot: whether the report adds detail no tool returned, states the differences plainly, and ends somewhere useful. Do not mark a report down for lacking per-fact tool attribution — the brief does not ask for it.", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ] } }, { "suite": "productivity", "id": "craftbook-powerpoint-deck", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "deck.md", "artifactKind": "markdown", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ] } }, { "suite": "productivity", "id": "wikipedia-research-brief", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "research-brief.md", "artifactKind": "markdown", "contextNote": "All historical evidence comes from the three signed local Wikipedia source cards. The “first programmer” label must remain calibrated rather than asserted as uncontested fact.", "axes": [ { "name": "synthesis", "description": "Builds an argument across sources rather than listing source summaries." }, { "name": "grounding", "description": "Claims and dates stay faithful to the signed local source cards." }, { "name": "citationQuality", "description": "Source ids appear near the claims they support and make provenance easy to audit." }, { "name": "calibration", "description": "Separates documented facts, interpretation, and disputed historical shorthand." } ] } }, { "suite": "productivity", "id": "craftbook-week-plan", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ] } }, { "suite": "productivity", "id": "craftbook-ab-test-readout", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "readout.md", "artifactKind": "markdown", "axes": [ { "name": "decision rule", "description": "The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift." }, { "name": "statistical reporting", "description": "Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json." }, { "name": "honest caveats", "description": "Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action." } ] } }, { "suite": "productivity", "id": "craftbook-annotated-bibliography", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "bibliography.md", "artifactKind": "markdown", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ] } }, { "suite": "productivity", "id": "craftbook-research-to-document", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ] } }, { "suite": "developer", "id": "fix-squisq-bugs", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "dev-craftbook-routing", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "interface-contract", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "craftbook-code-review", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "developer", "id": "craftbook-api-contract-review", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "api-review.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "developer", "id": "craftbook-deep-security-review", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "REPORT.md", "artifactKind": "markdown", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ] } }, { "suite": "developer", "id": "large-pr-review", "timeoutMs": 3600000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "craftbook-codemod-sweep", "timeoutMs": 5400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "verification.md", "artifactKind": "markdown", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ] } }, { "suite": "developer", "id": "craftbook-refactor-module", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "change-notes.md", "artifactKind": "markdown", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ] } }, { "suite": "developer", "id": "craftbook-bug-fix-tdd", "timeoutMs": 9000000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "fix-notes.md", "artifactKind": "markdown", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ] } }, { "suite": "complex-work", "id": "craftbook-find-vs-create", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-edit-midtask", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-invoice-run", "timeoutMs": 3000000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ] } }, { "suite": "complex-work", "id": "craftbook-author-linear", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-route-multi", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-export-generalize", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-params", "timeoutMs": 5400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-gate-script", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-fanout", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null } ], "evaluator": "standard-scorecard", "runtimeLayer": "required" }, "note": "Original condition: 85 terminal trials, 71 passes, 13 eligible failures and one sandbox-unavailable exclusion. Saluki 27B IQ2 Mix, native 0.1.57 CUDA on a 12 GB RTX 5070 Ti Laptop GPU. Every attempted task used hybrid placement: 39/64 dense FFN layers on GPU and 25 on CPU, context 65,536, one slot, f16 KV. The complete archive contains 128 of 129 planned attempts: 98 fixed passes, 27 eligible failures, one infrastructure exclusion, two unscored interruptions and one unattempted entry. Different environments and execution segments remain separate; three eligible repeats are not available for every entry. Task measurements are engine-log observations, not admission probes. Sonnet 5.5 reviews are advisory and use separate declared rubrics; original Codex reviews and fixed scores remain in the download." }, { "runId": "2026-11-06-windows-saluki-native-0-1-57-fanout", "segment": "fanout", "environmentHash": "0e697bb7b59c0b29a37cdbfaabd713db3215bb885648ff1fba70de15ba8d5a28", "definition": { "version": 1, "count": 1, "suites": [ "core", "productivity", "developer", "complex-work" ], "generalist": "auto", "cliAuth": "login", "targets": [ { "id": "saluki-cuda", "provider": "llama-cpp", "model": "underdog-saluki-27b-iq2-mix" } ], "sourceHome": "\\.gezel-dev", "scenarios": [ { "suite": "core", "id": "tictactoe", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "petshop", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": "sdxl-lightning-4step", "judge": null }, { "suite": "core", "id": "tankcombat", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "schema-migration", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "failing-tests-spec", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [ "vitest" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "symptom-debug", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "data-wrangle", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "incident-postmortem", "timeoutMs": 2400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "ops-runbook-anomaly", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "plan-and-estimate", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "conflict-synthesis", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "constrained-comms", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "records-intake", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "plan-and-estimate", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "meeting-followup", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "meeting-brief.md", "artifactKind": "markdown", "contextNote": "The August 27 transcript is authoritative; old-agenda.md contains rejected proposals that must not become decisions.", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ] } }, { "suite": "productivity", "id": "craftbook-spreadsheet-model", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "scope.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "productivity", "id": "conflict-synthesis", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "docblocks-theme-roundtrip", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "theme-report.md", "artifactKind": "markdown", "contextNote": "Theme inference returned exactly these values: theme id custom-brand-reference-pptx; heading font Kelder Grotesk; body font Marne Text; primary #2E5E4E; background #F4F1EA. The slide count and comparison figures depend on the session's own conversion, and a deterministic gate already checks them against the saved deck and the comparison result. Judge what it cannot: whether the report adds detail no tool returned, states the differences plainly, and ends somewhere useful. Do not mark a report down for lacking per-fact tool attribution — the brief does not ask for it.", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ] } }, { "suite": "productivity", "id": "craftbook-powerpoint-deck", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "deck.md", "artifactKind": "markdown", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ] } }, { "suite": "productivity", "id": "wikipedia-research-brief", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "research-brief.md", "artifactKind": "markdown", "contextNote": "All historical evidence comes from the three signed local Wikipedia source cards. The “first programmer” label must remain calibrated rather than asserted as uncontested fact.", "axes": [ { "name": "synthesis", "description": "Builds an argument across sources rather than listing source summaries." }, { "name": "grounding", "description": "Claims and dates stay faithful to the signed local source cards." }, { "name": "citationQuality", "description": "Source ids appear near the claims they support and make provenance easy to audit." }, { "name": "calibration", "description": "Separates documented facts, interpretation, and disputed historical shorthand." } ] } }, { "suite": "productivity", "id": "craftbook-week-plan", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ] } }, { "suite": "productivity", "id": "craftbook-ab-test-readout", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "readout.md", "artifactKind": "markdown", "axes": [ { "name": "decision rule", "description": "The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift." }, { "name": "statistical reporting", "description": "Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json." }, { "name": "honest caveats", "description": "Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action." } ] } }, { "suite": "productivity", "id": "craftbook-annotated-bibliography", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "bibliography.md", "artifactKind": "markdown", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ] } }, { "suite": "productivity", "id": "craftbook-research-to-document", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ] } }, { "suite": "developer", "id": "fix-squisq-bugs", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "dev-craftbook-routing", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "interface-contract", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "craftbook-code-review", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "developer", "id": "craftbook-api-contract-review", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "api-review.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "developer", "id": "craftbook-deep-security-review", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "REPORT.md", "artifactKind": "markdown", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ] } }, { "suite": "developer", "id": "large-pr-review", "timeoutMs": 3600000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "craftbook-codemod-sweep", "timeoutMs": 5400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "verification.md", "artifactKind": "markdown", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ] } }, { "suite": "developer", "id": "craftbook-refactor-module", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "change-notes.md", "artifactKind": "markdown", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ] } }, { "suite": "developer", "id": "craftbook-bug-fix-tdd", "timeoutMs": 9000000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "fix-notes.md", "artifactKind": "markdown", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ] } }, { "suite": "complex-work", "id": "craftbook-find-vs-create", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-edit-midtask", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-invoice-run", "timeoutMs": 3000000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ] } }, { "suite": "complex-work", "id": "craftbook-author-linear", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-route-multi", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-export-generalize", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-params", "timeoutMs": 5400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-gate-script", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-fanout", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null } ], "evaluator": "standard-scorecard", "runtimeLayer": "required" }, "note": "Separate logical repetition-2 fanout attempt, interrupted without result, facts, score or final metrics. It is administratively excluded, never retried or scored as a model failure. The continuation points GEZEL_MXC_EXEC at the existing verified Microsoft-signed sandbox launcher. Saluki 27B IQ2 Mix, native 0.1.57 CUDA on a 12 GB RTX 5070 Ti Laptop GPU. Every attempted task used hybrid placement: 39/64 dense FFN layers on GPU and 25 on CPU, context 65,536, one slot, f16 KV. The complete archive contains 128 of 129 planned attempts: 98 fixed passes, 27 eligible failures, one infrastructure exclusion, two unscored interruptions and one unattempted entry. Different environments and execution segments remain separate; three eligible repeats are not available for every entry. Task measurements are engine-log observations, not admission probes. Sonnet 5.5 reviews are advisory and use separate declared rubrics; original Codex reviews and fixed scores remain in the download." }, { "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "environmentHash": "0e697bb7b59c0b29a37cdbfaabd713db3215bb885648ff1fba70de15ba8d5a28", "definition": { "version": 1, "count": 1, "suites": [ "core", "productivity", "developer", "complex-work" ], "generalist": "auto", "cliAuth": "login", "targets": [ { "id": "saluki-cuda", "provider": "llama-cpp", "model": "underdog-saluki-27b-iq2-mix" } ], "sourceHome": "\\.gezel-dev", "scenarios": [ { "suite": "core", "id": "tictactoe", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "petshop", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": "sdxl-lightning-4step", "judge": null }, { "suite": "core", "id": "tankcombat", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [ "chromium" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "schema-migration", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "failing-tests-spec", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [ "vitest" ], "imageModel": null, "judge": null }, { "suite": "core", "id": "symptom-debug", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "data-wrangle", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "incident-postmortem", "timeoutMs": 2400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "ops-runbook-anomaly", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "plan-and-estimate", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "core", "id": "conflict-synthesis", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "constrained-comms", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "records-intake", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "plan-and-estimate", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "meeting-followup", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "meeting-brief.md", "artifactKind": "markdown", "contextNote": "The August 27 transcript is authoritative; old-agenda.md contains rejected proposals that must not become decisions.", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ] } }, { "suite": "productivity", "id": "craftbook-spreadsheet-model", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "scope.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "productivity", "id": "conflict-synthesis", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "productivity", "id": "docblocks-theme-roundtrip", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "theme-report.md", "artifactKind": "markdown", "contextNote": "Theme inference returned exactly these values: theme id custom-brand-reference-pptx; heading font Kelder Grotesk; body font Marne Text; primary #2E5E4E; background #F4F1EA. The slide count and comparison figures depend on the session's own conversion, and a deterministic gate already checks them against the saved deck and the comparison result. Judge what it cannot: whether the report adds detail no tool returned, states the differences plainly, and ends somewhere useful. Do not mark a report down for lacking per-fact tool attribution — the brief does not ask for it.", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ] } }, { "suite": "productivity", "id": "craftbook-powerpoint-deck", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "deck.md", "artifactKind": "markdown", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ] } }, { "suite": "productivity", "id": "wikipedia-research-brief", "timeoutMs": 2100000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "research-brief.md", "artifactKind": "markdown", "contextNote": "All historical evidence comes from the three signed local Wikipedia source cards. The “first programmer” label must remain calibrated rather than asserted as uncontested fact.", "axes": [ { "name": "synthesis", "description": "Builds an argument across sources rather than listing source summaries." }, { "name": "grounding", "description": "Claims and dates stay faithful to the signed local source cards." }, { "name": "citationQuality", "description": "Source ids appear near the claims they support and make provenance easy to audit." }, { "name": "calibration", "description": "Separates documented facts, interpretation, and disputed historical shorthand." } ] } }, { "suite": "productivity", "id": "craftbook-week-plan", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ] } }, { "suite": "productivity", "id": "craftbook-ab-test-readout", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "readout.md", "artifactKind": "markdown", "axes": [ { "name": "decision rule", "description": "The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift." }, { "name": "statistical reporting", "description": "Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json." }, { "name": "honest caveats", "description": "Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action." } ] } }, { "suite": "productivity", "id": "craftbook-annotated-bibliography", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "bibliography.md", "artifactKind": "markdown", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ] } }, { "suite": "productivity", "id": "craftbook-research-to-document", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ] } }, { "suite": "developer", "id": "fix-squisq-bugs", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "dev-craftbook-routing", "timeoutMs": 1500000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "interface-contract", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "craftbook-code-review", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "developer", "id": "craftbook-api-contract-review", "timeoutMs": 1800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "api-review.md", "artifactKind": "markdown", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "contextNote": "Advisory only. Deterministic pass/fail comes from the craftbook runtime gates, deliverable existence, attribution, and terminal progress." } }, { "suite": "developer", "id": "craftbook-deep-security-review", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "REPORT.md", "artifactKind": "markdown", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ] } }, { "suite": "developer", "id": "large-pr-review", "timeoutMs": 3600000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "developer", "id": "craftbook-codemod-sweep", "timeoutMs": 5400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "verification.md", "artifactKind": "markdown", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ] } }, { "suite": "developer", "id": "craftbook-refactor-module", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "change-notes.md", "artifactKind": "markdown", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ] } }, { "suite": "developer", "id": "craftbook-bug-fix-tdd", "timeoutMs": 9000000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "fix-notes.md", "artifactKind": "markdown", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ] } }, { "suite": "complex-work", "id": "craftbook-find-vs-create", "timeoutMs": 1200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-edit-midtask", "timeoutMs": 2700000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-invoice-run", "timeoutMs": 3000000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": { "artifactBasename": "report.md", "artifactKind": "markdown", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ] } }, { "suite": "complex-work", "id": "craftbook-author-linear", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-route-multi", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-export-generalize", "timeoutMs": 4800000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-params", "timeoutMs": 5400000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-gate-script", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null }, { "suite": "complex-work", "id": "craftbook-author-fanout", "timeoutMs": 7200000, "repairPolicy": "harness", "requires": [], "imageModel": null, "judge": null } ], "evaluator": "standard-scorecard", "runtimeLayer": "required" }, "note": "Separate continuation condition: 41 terminal trials (27 passes, 14 eligible failures), one unscored interrupted gate-script attempt, one unattempted fanout. Local r1 cell IDs map to original logical r3. The sole environment change from the original is GEZEL_MXC_EXEC pointing at the existing verified Microsoft-signed sandbox launcher. The stopped journal's stale running field remains intact. Saluki 27B IQ2 Mix, native 0.1.57 CUDA on a 12 GB RTX 5070 Ti Laptop GPU. Every attempted task used hybrid placement: 39/64 dense FFN layers on GPU and 25 on CPU, context 65,536, one slot, f16 KV. The complete archive contains 128 of 129 planned attempts: 98 fixed passes, 27 eligible failures, one infrastructure exclusion, two unscored interruptions and one unattempted entry. Different environments and execution segments remain separate; three eligible repeats are not available for every entry. Task measurements are engine-log observations, not admission probes. Sonnet 5.5 reviews are advisory and use separate declared rubrics; original Codex reviews and fixed scores remain in the download." } ], "identityComparison": { "at": "2026-11-06T19:53:52.538Z", "originalEvidenceUnchanged": true, "identityChangedKeys": [ "environmentHash" ], "originalEnvironmentHash": "688bb13249f9e1905a35004e7b522f2c9b7ed05aa404a535be89f6069d61d44d", "newEnvironmentHash": "0e697bb7b59c0b29a37cdbfaabd713db3215bb885648ff1fba70de15ba8d5a28", "otherIdentityComponentsEqual": true, "launcherSha256": "8a6d1db4dd1846981127d9014741be18b094d07be967765dff6754bda1d40259", "remaining": 44 }, "nativeVerification": { "release": "0.1.57", "sourceManifest": "packages/inference/src/engines/native-file-manifest.json", "installRoot": "packages/app/native-bin", "verifiedFiles": 62, "verifiedPlatformKeys": [ "win32-x64", "win32-x64-cpu", "win32-x64-vulkan", "win32-x64-cuda" ], "cudaArchiveSha256": "9e3f21713fe77c23536dd4124244ac3834f90c82d6e470cb6836a73bb4f7c592", "llamaVersion": "0.8.0", "llamaBuild": 11960, "llamaRevision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "model": "underdog-saluki-27b-iq2-mix", "modelBytes": 7898369152, "modelVerifiedSha256": "4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb", "gpu": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "gpuMemoryMiB": 12227, "driverVersion": "610.88", "chromiumVersion": "156.0.8078.4", "inferenceAndServiceBuild": "passed", "electronPackagingSmoke": "failed inside sandbox during bundled pnpm clone test; not used by eval daemon" }, "trials": [ { "cellId": "r1-saluki-cuda-core-tictactoe", "baselineCellId": "r1-saluki-cuda-core-tictactoe", "suite": "core", "scenario": "tictactoe", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T00:05:25.966Z", "finishedAt": "2026-11-04T00:13:25.423Z", "durationMs": 479457, "success": true, "reason": "tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 465554 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 3826, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": 2, "runtimeFailed": 0 } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 3826 } ], "adjustments": [ { "ruleId": "Q_HTML_REALISM_CAP", "summary": "Primary game HTML is 3826 bytes, below the 4096-byte realism floor; quality is capped at 5." } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.03199709617828312 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 465554 }, { "source": "facts", "path": "outcome.durationMs", "value": 479457 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.8, "band": "ship-ready" }, "facts": { "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 79, "peakRssMb": 11404, "peakCpuPercent": 1239.2, "firstAtMs": 1, "lastAtMs": 460813 }, "gpu": { "available": true, "sampleCount": 79, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 79, "peakUsedMb": 30088, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 13337, "totalOutputTokens": 3600, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 22396, "todayTokensCached": 21204, "todayTokensOut": 1791, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 22396, "totalTokensCached": 21204, "totalTokensOut": 1791, "totalCost": 0, "medianOutputTokensPerSec": 9.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.2, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7696, "cachedInputTokens": 7422, "outputTokens": 1306, "cost": 0, "durationMs": 157329, "outputTokensPerSec": 8.411167516780036, "at": "2026-11-04T00:10:21.721Z" }, "lastUpdated": "2026-11-04T00:10:21.721Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T00:13:16.004Z" } }, "source": "engine-log" }, "derived": { "durationMs": 464831, "meanTokensPerSec": 8.68, "genTokensPerSec": 8.68, "genTokens": 3600, "promptTokensPerSec": 475.25, "peakRssMb": 11404, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22396, "output": 1791 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tic-tac-toe-game/1", "stepId": "build", "activatedAt": "2026-11-04T00:06:47.720Z", "completedAt": "2026-11-04T00:13:12.430Z", "ms": 384710, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "20be0f5d-7075-4e14-b49b-03adcb6282c1" ] }, { "ref": "tic-tac-toe-game/1", "stepId": "evaluate", "activatedAt": "2026-11-04T00:13:12.432Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T00:05:31.202Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T00:05:31.087Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 384710 }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "daniyar": 2, "yusuf": 1 }, "perTask": { "tic-tac-toe-game/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.25, "maxMessages": 6, "maxContextFill": 0.1108856201171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1108856201171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "durationMs": 479457, "timeoutMs": 14984391, "budgetUsedFraction": 0.03199709617828312 }, "timing": { "startedAt": "2026-11-04T00:05:25.966Z", "finishedAt": "2026-11-04T00:13:25.423Z", "timeToFirstArtifactMs": 465554, "timeToLastArtifactWriteMs": 465612, "timeToFirstTokenMs": 27373, "firstTurnTtftMs": 18536, "timeToFirstToolCallMs": 81809 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "start_work": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/tic-tac-toe-game/index.html", "finalBytes": 3826, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 467056, "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 3826, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null } ], "latest": { "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 3826, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": 2, "runtimeFailed": 0 } }, "autoAnswer": { "total": 1, "byKind": { "structured": 0, "inline": 1 }, "events": [ { "atMs": 330837, "kind": "inline", "gezel": "meester", "question": null, "chose": "default" } ] }, "miscEvents": [ "[2026-11-04T00:05:25.969Z] [trial] id=tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek scenario=tictactoe model=underdog-saluki-27b-iq2-mix", "[2026-11-04T00:05:25.969Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "[2026-11-04T00:05:25.969Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T00:05:25.969Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-tictactoe-BNYDlf", "[2026-11-04T00:05:25.969Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T00:05:25.970Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T00:05:25.973Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T00:05:25.977Z] [trial] throughput-scaled ceiling: 120m → 250m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T00:05:25.976Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-tictactoe-BNYDlf", "[2026-11-04T00:05:31.130Z] [trial] daemon spawned pid=36676 port=https://127.0.0.1:50647", "[2026-11-04T00:05:32.009Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T00:05:32.026Z] [trial] meester=yusuf", "[2026-11-04T00:05:32.171Z] [trial] sent prompt to meester (621 chars)", "[2026-11-04T00:05:32.222Z] [poll] initial digests hard=6e50d2221fe3bebc soft=8e1b7d17ee7c2f6b", "[2026-11-04T00:06:49.052Z] [poll] hard-progress: 6e50d2221fe3bebc → 62aa9ce039b2f723 after 77s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=168,stream=644) sniff=none)", "[2026-11-04T00:07:53.987Z] [poll] hard-progress: 62aa9ce039b2f723 → a736cd66d2b3412d after 65s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=1,slot=353,stream=1188) sniff=none)", "[2026-11-04T00:11:55.887Z] [poll] hard-progress: a736cd66d2b3412d → 7ac6d010288210df after 242s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=1,slot=982,stream=3558) sniff=none)", "[2026-11-04T00:13:15.693Z] [poll] terminal: success=true reason=tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "[2026-11-04T00:13:16.045Z] [capture] dumped 3 session(s)", "[2026-11-04T00:13:16.048Z] [capture] snapshotted telemetry for 3 session(s)", "[2026-11-04T00:13:16.098Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T00:13:25.423Z] [trial] daemon shut down", "[2026-11-04T00:13:25.432Z] [trial] PASS duration=479457ms reason=tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))" ] }, "metrics": { "process": { "sampleCount": 79, "peakRssMb": 11404, "peakCpuPercent": 1239.2, "firstAtMs": 1, "lastAtMs": 460813 }, "gpu": { "available": true, "sampleCount": 79, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 79, "peakUsedMb": 30088, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 13337, "totalOutputTokens": 3600, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 22396, "todayTokensCached": 21204, "todayTokensOut": 1791, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 22396, "totalTokensCached": 21204, "totalTokensOut": 1791, "totalCost": 0, "medianOutputTokensPerSec": 9.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.2, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7696, "cachedInputTokens": 7422, "outputTokens": 1306, "cost": 0, "durationMs": 157329, "outputTokensPerSec": 8.411167516780036, "at": "2026-11-04T00:10:21.721Z" }, "lastUpdated": "2026-11-04T00:10:21.721Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T00:13:16.004Z" } }, "source": "engine-log" }, "derived": { "durationMs": 464831, "meanTokensPerSec": 8.68, "genTokensPerSec": 8.68, "genTokens": 3600, "promptTokensPerSec": 475.25, "peakRssMb": 11404, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22396, "output": 1791 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T00:05:25.966Z", "finishedAt": "2026-11-04T00:13:25.423Z", "timeToFirstArtifactMs": 465554, "timeToLastArtifactWriteMs": 465612, "timeToFirstTokenMs": 27373, "firstTurnTtftMs": 18536, "timeToFirstToolCallMs": 81809 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tic-tac-toe-game/1", "stepId": "build", "activatedAt": "2026-11-04T00:06:47.720Z", "completedAt": "2026-11-04T00:13:12.430Z", "ms": 384710, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "20be0f5d-7075-4e14-b49b-03adcb6282c1" ] }, { "ref": "tic-tac-toe-game/1", "stepId": "evaluate", "activatedAt": "2026-11-04T00:13:12.432Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T00:05:31.202Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T00:05:31.087Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 384710 }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "daniyar": 2, "yusuf": 1 }, "perTask": { "tic-tac-toe-game/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.25, "maxMessages": 6, "maxContextFill": 0.1108856201171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1108856201171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-tictactoe", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "scenarioId": "tictactoe", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "axisSource": "default HTML axes in evals/src/llm-judge.ts; the legacy HTML scenario does not declare overrides", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/workspace/tic-tac-toe-game/index.html", "artifactBytes": 3826, "previewPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/judge-preview.png", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 7, "codeQuality": 7, "polish": 5 }, "meanScore": 6, "justification": "Clean centered board, explicit turns/wins, draw handling and reset. A small readable state machine blocks invalid moves. Styling is generic; fixed cell widths and missing cell labels/live status limit mobile and accessibility polish.", "reviewBasis": "Full artifact source within 20 KiB, rendered desktop preview, and recorded two-assertion runtime pass. Model identity was visible in evidence paths." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:54:50.975Z", "reviewId": "001", "cellId": "r1-saluki-cuda-core-tictactoe", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "scenarioId": "tictactoe", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "baselineCellId": "r1-saluki-cuda-core-tictactoe", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "cb35e3d088759a4362406e42dc27797a7798258d37e13820807b5b3301f5dd6e", "evidenceSha256": "1f8789319e361dd3394780fb421f742ca00f7bfef129d7b2864d434d90954b29", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/workspace/tic-tac-toe-game/index.html", "bytes": 3826, "sha256": "e9b569395a328350da51419c7b119c76a20fe7c503d378dc05108b34dd5163f5", "shownBytes": 3826, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 8, "codeQuality": 7, "polish": 4 }, "justification": "Meets the brief: nine static .cell buttons with data-cell 0–8, wired via querySelectorAll, alternating turns, win and draw detection, and a New Game button. The file sits in a new project folder. Code is clean and readable. The look is clean but plain: grid, hover, X/O colours, green winning line. Few extras.", "limitations": "Source-only review with no rendering or execution. Layout and behaviour are inferred from the code. The path is workspace/tic-tac-toe-game/index.html, not the literal workspace/index.html, which is a reasonable reading of the \"new project\" instruction.", "meanScore": 6, "durationMs": 16182, "costUsd": 0.023252, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4412, "cache_read_input_tokens": 0, "output_tokens": 560, "output_tokens_details": { "thinking_tokens": 212 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4412, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 560, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 4412, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4412 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 560, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 4412, "webSearchRequests": 0, "costUSD": 0.023252, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 212, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4412, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - tictactoe / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.8 / 10** (band: ship-ready)\n**Outcome: success** (duration: 7m 59s; budget used: 3.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 5 / 10 | The scenario gate is complete. Primary game HTML is 3826 bytes, below the 4096-byte realism floor; quality is capped at 5. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 465554`; `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":3826,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`; `facts.sniff.latest.bytes: 3826` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.03199709617828312`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 465554`; `facts.outcome.durationMs: 479457` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **8.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11404 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30088 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.68 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 13337 + 3600 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 4 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 385s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.25 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 11% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- Adjustment Q_HTML_REALISM_CAP: Primary game HTML is 3826 bytes, below the 4096-byte realism floor; quality is capped at 5.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 465554`\n- `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":3826,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`\n- `facts.sniff.latest.bytes: 3826`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.03199709617828312`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 465554`\n- `facts.outcome.durationMs: 479457`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 465554`; `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":3826,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`; `facts.sniff.latest.bytes: 3826`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.03199709617828312`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 465554`; `facts.outcome.durationMs: 479457`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Inline qualitative review\n\nAdvisory only: codex-inline/current-session, default HTML axes, complete 3826-byte source plus a rendered 900-by-700 preview. Visual quality 5, functional completeness 7, code quality 7, polish 5; mean 6.0/10. This does not change the fixed 8.8 composite. The centered board is readable, the state machine covers wins/draws/reset and rejects invalid moves, and the styling is basic. Fixed cell widths and absent cell labels/live status limit mobile and accessibility polish. The campaign's `codex-inline-judge.json` contains the exact axes and evidence path.\n\n## Positives and limitations\n\n- The game passed both static and browser checks, with no recorded tool red flags or missing roles (`facts.outcome.success`, `facts.outcome.reason`, `facts.toolUse.redFlags`, `facts.team.missingExpectedRoles`).\n- The model completed this task in 479457 ms and decoded 3600 tokens at 8.68 tokens/sec (`facts.outcome.durationMs`, `facts.perf.derived.genTokens`, `facts.perf.derived.genTokensPerSec`). The first artifact arrived after 465554 ms (`facts.timing.timeToFirstArtifactMs`).\n- Peak VRAM was 10628/12227 MiB, process-tree RSS 11404 MiB and total system memory use 30088/31865 MiB (`facts.perf.gpu`, `facts.perf.process`, `facts.perf.systemMemory`). Whole-system use includes other applications. The launch used a 64K f16 cache and moved FFN computation for 25/64 layers to CPU (`result.engineContext`, `daemon.log` offload plan). This is mixed CPU/GPU execution, not a full GPU-residency result.\n\n## Tactical experiment\n\nIn Gilde's `data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json`, test `tuning.reasoning.thinkingBudget` and `tuning.profiles.thinking-general.reasoning.thinkingBudget` at 2048 rather than 4096 in a separate baseline. The motivation is measured decode cost and time to first artifact, not a proven reasoning defect (`facts.perf.derived.genTokens: 3600`, `facts.timing.timeToFirstArtifactMs: 465554`). Check at least a coding and a planning scenario before recommending it; fewer reasoning tokens could hurt correctness. No tuning was changed in this campaign.\n\n## Strategic experiment\n\nInspect `packages/inference/src/providers/native/capacity-broker.ts` and the llama.cpp offload planner to compare f16 and q8_0 KV plans at the same 65536-token window before assigning dense FFNs to CPU. Measure whether q8_0 reduces CPU FFN placement and improves sustained decode without OOM or tool-accuracy loss. The relevant evidence is the 25-layer CPU FFN plan and the measured 8.68 tokens/sec, not the HTML score. This is an experiment, not a validated fix; preserve the current baseline and compare across unrelated tasks.\n\n## Generalization and historical comparison\n\nNo prompts, model settings, scenario gates or artifacts were altered. The proposed experiments concern general decode cost and memory allocation. The unrelated petshop trial is still active and is not scored here; its tool-repair behavior cannot yet support a final cross-scenario conclusion.\n\nThe last three locally available completed tictactoe facts are the Gemma 4 12B/native 0.1.54 trials under `evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tictactoe/`, starting at 2026-10-28T07:05:12.793Z, 07:06:03.125Z and 07:06:54.917Z. They report composites of 10 and durations of 49.1-51.4 seconds, but each explicitly says `runtime layer unavailable`. This Saluki trial includes the runtime assertions. Different model/native release and unequal runtime validation prevent treating the score difference as a controlled regression. Do not infer a pass rate from this single completed Saluki trial.\n\n## Recommended next experiment\n\nAfter the frozen campaign finishes, compare the unchanged f16 plan against q8_0 KV at the same 64K context on at least a coding and a planning scenario, measuring CPU FFN placement, sustained decode, peak VRAM/RAM and tool correctness.\n" }, { "cellId": "r1-saluki-cuda-core-petshop", "baselineCellId": "r1-saluki-cuda-core-petshop", "suite": "core", "scenario": "petshop", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T00:16:26.052Z", "finishedAt": "2026-11-04T01:37:29.262Z", "durationMs": 4863210, "success": true, "reason": "pet-shop-website/workspace/index.html passed sniff (signals: pet-vocab, store-vocab, structured-page, working-image, image-asset) + runtime (1 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "requiredRasterMinBytes": 51200 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 737836 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "pet-shop-website/workspace/index.html", "bytes": 5780, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": 1, "runtimeFailed": 0 } }, { "source": "facts", "path": "artifacts.imageFiles", "value": [ { "path": "workspace/pet-shop-website/assets/logo.png", "bytes": 3418137 }, { "path": "artifacts/pet-shop-website/generated/image-2026-11-04T01-37-09-941Z-1379962518.png", "bytes": 3418137 }, { "path": "artifacts/pet-shop-website/sessions/2026-11-04_002848_new-session/tool-0-img-0.png", "bytes": 3418137 } ] } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.32455172852870695 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 27 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 737836 }, { "source": "facts", "path": "outcome.durationMs", "value": 4863210 } ] }, "behavior": { "score": 0, "ruleId": "B_RED_FLAG_OR_MISSING_ROLE", "summary": "At least one red-flag behavior fired or an expected role was missing.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [ "image-generator" ] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 7.5, "band": "needs-tuning" }, "facts": { "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 811, "peakRssMb": 13753, "peakCpuPercent": 1712.8, "firstAtMs": 0, "lastAtMs": 4842089 }, "gpu": { "available": true, "sampleCount": 811, "peakUtilPercent": 100, "peakMemUsedMb": 10654, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 811, "peakUsedMb": 31525, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 87916, "totalOutputTokens": 24320, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 35257, "todayTokensCached": 33930, "todayTokensOut": 505, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 35257, "totalTokensCached": 33930, "totalTokensOut": 505, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8115, "cachedInputTokens": 7856, "outputTokens": 197, "cost": 0, "durationMs": 767354, "outputTokensPerSec": 9.044884222713108, "at": "2026-11-04T00:31:35.578Z" }, "lastUpdated": "2026-11-04T00:31:35.578Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T01:37:16.152Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4844649, "meanTokensPerSec": 5.54, "genTokensPerSec": 5.54, "genTokens": 24320, "promptTokensPerSec": 314.11, "peakRssMb": 13753, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35257, "output": 505 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "pet-shop-website/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-04T00:18:29.141Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "aa80d5fb-90cc-4576-804d-25c9734f61c3" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T00:16:31.584Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T00:16:31.487Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 7, "taskScoped": 1, "byGezel": { "adama": 1, "cassian": 2, "hua": 1, "otgonbayar": 2, "wilma": 1 }, "perTask": { "pet-shop-website/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.264862060546875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.264862060546875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 3 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "pet-shop-website/workspace/index.html passed sniff (signals: pet-vocab, store-vocab, structured-page, working-image, image-asset) + runtime (1 assertion(s))", "durationMs": 4863210, "timeoutMs": 14984391, "budgetUsedFraction": 0.32455172852870695 }, "timing": { "startedAt": "2026-11-04T00:16:26.052Z", "finishedAt": "2026-11-04T01:37:29.262Z", "timeToFirstArtifactMs": 737836, "timeToLastArtifactWriteMs": 4643401, "timeToFirstTokenMs": 27337, "firstTurnTtftMs": 18299, "timeToFirstToolCallMs": 123144 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer", "Builder", "Illustrator" ], "missingExpectedRoles": [ "image-generator" ] }, "toolUse": { "totalToolCalls": 27, "byTool": { "start_work": 1, "save_memory": 1, "message_gezel": 4, "ensure_gezel": 2, "write_file": 8, "run_installed_script": 1, "read_file": 2, "generate_image": 1, "delegate_image_generator": 1, "delegate_developer": 1, "list_dir": 1, "list_artifacts": 1, "stat": 1, "list_project_gezels": 1, "read_task_notes": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/pet-shop-website/index.html", "finalBytes": 5788, "growth": [] } ], "imageFiles": [ { "path": "workspace/pet-shop-website/assets/logo.png", "bytes": 3418137 }, { "path": "artifacts/pet-shop-website/generated/image-2026-11-04T01-37-09-941Z-1379962518.png", "bytes": 3418137 }, { "path": "artifacts/pet-shop-website/sessions/2026-11-04_002848_new-session/tool-0-img-0.png", "bytes": 3418137 } ], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 742667, "filePath": "pet-shop-website/workspace/index.html", "bytes": 5780, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 4848647, "filePath": "pet-shop-website/workspace/index.html", "bytes": 5780, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null } ], "latest": { "filePath": "pet-shop-website/workspace/index.html", "bytes": 5780, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": 1, "runtimeFailed": 0 } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T00:16:26.056Z] [trial] id=petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w scenario=petshop model=underdog-saluki-27b-iq2-mix", "[2026-11-04T00:16:26.056Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "[2026-11-04T00:16:26.056Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-petshop-7GP0lZ", "[2026-11-04T00:16:26.056Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T00:16:26.058Z] [trial] sd-server=packages\\app\\native-bin\\win32-x64\\gezel-sd-server.exe", "[2026-11-04T00:16:26.056Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T00:16:26.057Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T00:16:26.061Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T00:16:26.063Z] [trial] sd-cpp source=\\.gezel-eval-cache\\engines\\sd-cpp\\models\\sdxl-lightning-4step (read-only)", "[2026-11-04T00:16:26.069Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-petshop-7GP0lZ", "[2026-11-04T00:16:26.073Z] [trial] cloned sd-cpp/sdxl-lightning-4step from \\.gezel-eval-cache\\engines\\sd-cpp\\models\\sdxl-lightning-4step into \\AppData\\Local\\Temp\\gezel-eval-petshop-7GP0lZ", "[2026-11-04T00:16:26.073Z] [trial] throughput-scaled ceiling: 120m → 250m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T00:16:31.534Z] [trial] daemon spawned pid=42828 port=https://127.0.0.1:60809", "[2026-11-04T00:16:32.457Z] [trial] provider=llama-cpp imageProvider=sd-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T00:16:32.473Z] [trial] meester=cassian", "[2026-11-04T00:16:32.639Z] [trial] sent prompt to meester (795 chars)", "[2026-11-04T00:16:32.699Z] [poll] initial digests hard=57b3d87fcfe6cae7 soft=c075575489af7a9c", "[2026-11-04T00:18:30.823Z] [poll] hard-progress: 57b3d87fcfe6cae7 → be6d48dd017fc003 after 118s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=293,stream=1101) sniff=none)", "[2026-11-04T00:19:35.664Z] [poll] hard-progress: be6d48dd017fc003 → fe0b4588fbb482b1 after 65s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=1,slot=484,stream=1661) sniff=none)", "[2026-11-04T00:21:58.093Z] [poll] hard-progress: fe0b4588fbb482b1 → 0e8335cd528b8312 after 142s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=1,slot=904,stream=2906) sniff=none)", "[2026-11-04T00:23:51.508Z] [poll] hard-progress: 0e8335cd528b8312 → edecfd9644909802 after 113s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=2,slot=1212,stream=3888) sniff=none)", "[2026-11-04T00:25:15.160Z] [poll] hard-progress: edecfd9644909802 → 1dc31cec014f2bf6 after 84s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=4,slot=1296,stream=4125) sniff=none)", "[2026-11-04T00:28:49.046Z] [poll] hard-progress: 1dc31cec014f2bf6 → d7008c576045903e after 214s (workspaceProjects=3 sessions=5 daemon[src=service-telemetry](turns=3,tools=5,slot=1858,stream=7211) sniff=3/5780B)", "[2026-11-04T00:29:48.426Z] [poll] hard-progress: d7008c576045903e → e321fc5e92628945 after 59s (workspaceProjects=3 sessions=6 daemon[src=service-telemetry](turns=5,tools=7,slot=1992,stream=7894) sniff=3/5780B)", "[2026-11-04T00:30:48.304Z] [poll] hard-progress: e321fc5e92628945 → 9ba7893b71497a32 after 60s (workspaceProjects=3 sessions=7 daemon[src=service-telemetry](turns=5,tools=8,slot=2141,stream=8435) sniff=3/5780B)" ] }, "metrics": { "process": { "sampleCount": 811, "peakRssMb": 13753, "peakCpuPercent": 1712.8, "firstAtMs": 0, "lastAtMs": 4842089 }, "gpu": { "available": true, "sampleCount": 811, "peakUtilPercent": 100, "peakMemUsedMb": 10654, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 811, "peakUsedMb": 31525, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 87916, "totalOutputTokens": 24320, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 35257, "todayTokensCached": 33930, "todayTokensOut": 505, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 35257, "totalTokensCached": 33930, "totalTokensOut": 505, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8115, "cachedInputTokens": 7856, "outputTokens": 197, "cost": 0, "durationMs": 767354, "outputTokensPerSec": 9.044884222713108, "at": "2026-11-04T00:31:35.578Z" }, "lastUpdated": "2026-11-04T00:31:35.578Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T01:37:16.152Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4844649, "meanTokensPerSec": 5.54, "genTokensPerSec": 5.54, "genTokens": 24320, "promptTokensPerSec": 314.11, "peakRssMb": 13753, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35257, "output": 505 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T00:16:26.052Z", "finishedAt": "2026-11-04T01:37:29.262Z", "timeToFirstArtifactMs": 737836, "timeToLastArtifactWriteMs": 4643401, "timeToFirstTokenMs": 27337, "firstTurnTtftMs": 18299, "timeToFirstToolCallMs": 123144 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "pet-shop-website/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-04T00:18:29.141Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "aa80d5fb-90cc-4576-804d-25c9734f61c3" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T00:16:31.584Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T00:16:31.487Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 7, "taskScoped": 1, "byGezel": { "adama": 1, "cassian": 2, "hua": 1, "otgonbayar": 2, "wilma": 1 }, "perTask": { "pet-shop-website/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.264862060546875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.264862060546875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 3 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-petshop", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "scenarioId": "petshop", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "axisSource": "default HTML axes in evals/src/llm-judge.ts; legacy HTML scenario without overrides", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/workspace/pet-shop-website/index.html", "artifactBytes": 5788, "previewPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/judge-preview.png", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 1, "functionalCompleteness": 4, "codeQuality": 3, "polish": 1 }, "meanScore": 2.25, "justification": "The generated PNG exists and the page has sensible sections, anchors and labeled fields. However, styles.css is missing, so a huge unconstrained logo fills the initial viewport and the site is unstyled. The contact form only displays a local thank-you message. It passes the gate but is not a finished website.", "reviewBasis": "Full 5788-byte HTML within 20 KiB, workspace dependency listing, rendered desktop preview and recorded browser gate. Model identity was visible in evidence paths." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:56:32.775Z", "reviewId": "002", "cellId": "r1-saluki-cuda-core-petshop", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "scenarioId": "petshop", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "baselineCellId": "r1-saluki-cuda-core-petshop", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "cc515a209dbe3e6f4a9f39867c2321a247aeac973b3f2a0187b33778ef55b47e", "evidenceSha256": "92e0d520b750c61400207c80f8919405eb083b0ae79420e0cc016b61053af12c", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/workspace/pet-shop-website/index.html", "bytes": 5788, "sha256": "74b854669c8c13c12f5a2ffd7b8709db645ca1506eb37953445aa63d0eda2dce", "shownBytes": 5788, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 6, "codeQuality": 7, "polish": 5 }, "justification": "index.html is clean, semantic and accessible. It has the exact logo img tag, a nav, hero, product and service cards, an about section, a contact form with a thank-you message, and a footer. Layout is credible, but there are no product images and the styling can't be seen. Polish is limited to the form feedback and the stats list.", "limitations": "Only index.html was supplied. styles.css and assets/logo.png were not shown, so the styling, the logo's existence and quality, and the image-generation tool use are unverified. No rendering was done.", "meanScore": 5.75, "durationMs": 17212, "costUsd": 0.0230556, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4254, "cache_read_input_tokens": 856, "output_tokens": 595, "output_tokens_details": { "thinking_tokens": 274 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4254, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 595, "cache_read_input_tokens": 856, "cache_creation_input_tokens": 4254, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4254 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 595, "cacheReadInputTokens": 856, "cacheCreationInputTokens": 4254, "webSearchRequests": 0, "costUSD": 0.0230556, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 274, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4254, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - petshop / underdog-saluki-27b-iq2-mix\n\n**Composite: 7.5 / 10** (band: needs-tuning)\n**Outcome: success** (duration: 81m 3s; budget used: 32.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 737836`; `facts.sniff.latest: {\"filePath\":\"pet-shop-website/workspace/index.html\",\"bytes\":5780,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fa...`; `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-website/assets/logo.png\",\"bytes\":3418137},{\"path\":\"artifacts/pet-shop-website/generated/image-2026-11-04T01-37-09-941Z-1379962518.png\",\"bytes\":34181...` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.32455172852870695`; `facts.toolUse.totalToolCalls: 27`; `facts.timing.timeToFirstArtifactMs: 737836`; `facts.outcome.durationMs: 4863210` |\n| Behavior soundness (15%) | 0 / 10 | At least one red-flag behavior fired or an expected role was missing. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: [\"image-generator\"]`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 5 + 0.15 * 0 = **7.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13753 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10654 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31525 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 5.54 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 87916 + 24320 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 26% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 3 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 737836`\n- `facts.sniff.latest: {\"filePath\":\"pet-shop-website/workspace/index.html\",\"bytes\":5780,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fa...`\n- `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-website/assets/logo.png\",\"bytes\":3418137},{\"path\":\"artifacts/pet-shop-website/generated/image-2026-11-04T01-37-09-941Z-1379962518.png\",\"bytes\":34181...`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.32455172852870695`\n- `facts.toolUse.totalToolCalls: 27`\n- `facts.timing.timeToFirstArtifactMs: 737836`\n- `facts.outcome.durationMs: 4863210`\n\n### Behavior soundness - B_RED_FLAG_OR_MISSING_ROLE\n\nAt least one red-flag behavior fired or an expected role was missing.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: [\"image-generator\"]`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 737836`; `facts.sniff.latest: {\"filePath\":\"pet-shop-website/workspace/index.html\",\"bytes\":5780,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fa...`; `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-website/assets/logo.png\",\"bytes\":3418137},{\"path\":\"artifacts/pet-shop-website/generated/image-2026-11-04T01-37-09-941Z-1379962518.png\",\"bytes\":34181...`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.32455172852870695`; `facts.toolUse.totalToolCalls: 27`; `facts.timing.timeToFirstArtifactMs: 737836`; `facts.outcome.durationMs: 4863210`\n- Behavior: At least one red-flag behavior fired or an expected role was missing. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: [\"image-generator\"]`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Inline qualitative review\n\nAdvisory only, codex-inline/current-session: visual quality 1, functional completeness 4, code quality 3, polish 1; mean 2.25/10. Reviewed the full 5788-byte HTML and a rendered 900-by-700 preview. The referenced styles.css is absent: the workspace contains only index.html and assets/logo.png. The unconstrained logo fills the initial viewport and the page is unstyled. Semantic sections, anchors and labeled fields exist; the form only displays a local thank-you message. The fixed 7.5 composite remains unchanged; its gate-based quality axis did not detect the missing stylesheet. Exact axes are in the campaign's codex-inline-judge.json.\n\n## Positives and limitations\n\n- The static and one browser assertion passed; a real 3418137-byte image was generated at the referenced path (`facts.outcome`, `facts.artifacts.imageFiles`, `facts.toolUse.byTool.generate_image: 1`).\n- Completion took 4863210 ms (81m 3s), with 27 tool calls, eight write_file calls and eight created gezels (`facts.outcome.durationMs`, `facts.toolUse`, `facts.team.totalGezelsCreated`). The same-model tictactoe trial needed two calls and 479457 ms. Multi-role asset coordination added substantial overhead here.\n- Wilma's saved session records repeated write_file calls with content `placeholder` for a PNG, each rejected with an explicit binary-format error. At 00:40:54Z daemon.log records the five-error abort and a generic replace_lines remedy, which cannot create a raster image. These attempts did not trigger a fixed red flag (`facts.toolUse.redFlags: []`).\n- Decode averaged 5.54 tokens/sec over 24320 tokens. Peak VRAM was 10654/12227 MiB, process RSS 13753 MiB, and whole-system use 31525/31865 MiB (`facts.perf`); the last measure includes other applications. At 01:36:05Z the text engine exited with expectedReason=memory-pressure as sd-server started. The image render completed in 62.60 seconds. This demonstrates engine switching under the memory policy, not simultaneous residency. No unexpected OOM/crash was found.\n\n## Tactical experiment\n\nIn Gilde's `data/gezel-templates/il/illustrator/versions/1.0.0/about.md`, test adding under Checking the work: \"Deliver an actual rendered image before reporting it ready; never substitute a text placeholder for an image file.\" This role-level experiment is grounded in the saved repeated placeholder writes and `facts.toolUse.byTool.write_file: 8`. Validate an unrelated image brief before recommending it. No content or tuning changed in this campaign.\n\n## Strategic experiments\n\n- In `packages/core/src/local-loop/tool-failure-tracker.ts`, classify deterministic binary-format errors separately. After two identical failures, preserve the actual tool error's valid-image remedy instead of generic source-edit guidance. Keep transient failures and ordinary text edits on their current policy. The recorded Wilma errors and 00:40:54Z abort show this mismatch.\n- In `evals/src/scenarios/helpers.ts`, consider a shared local stylesheet/script dependency check for HTML in a future evaluator baseline. The absent CSS is a real output defect missed by the existing raster gate. Preserve this run's pass status and gate.\n- In `evals/src/bin/score-trial.ts`, investigate canonical role identity in missingScenarioRoles instead of display-name substring matching. `facts.team.rolesCreated` contains Illustrator and the image tool ran, but missingExpectedRoles contains image-generator. Leave the frozen behavior penalty intact; it does not prove that no image specialist existed.\n\n## Generalization and strategic note\n\nTictactoe produced a self-contained page without missing CSS or repeated binary-write errors. The reusable issues here are dependency completeness, choosing a valid operation after a deterministic error, and coordination on one engine slot. Do not encode scenario names or expected filenames in model prompts.\n\nThe last three local petshop facts are Gemma 4 12B/native 0.1.54 trials starting 2026-10-28T07:07:48.402Z, 07:11:10.713Z and 07:13:04.369Z under the October 27 Windows scorecard. Their fixed composites are 8.0, 8.5 and 8.5; durations 201.9, 112.8 and 147.7 seconds. All report runtime layer unavailable and missing image-generator. Different models/releases and unequal browser validation prevent a controlled regression claim. The recurring role penalty warrants evaluator identity review rather than assuming repeated recruitment failure.\n\n## Recommended next experiment\n\nAfter the frozen campaign, test two-identical-failure recovery for deterministic binary-format errors across an image brief and an unrelated binary-artifact task, measuring invalid repeats, latency and successful deliverables without changing gates.\n" }, { "cellId": "r1-saluki-cuda-core-tankcombat", "baselineCellId": "r1-saluki-cuda-core-tankcombat", "suite": "core", "scenario": "tankcombat", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T01:40:41.738Z", "finishedAt": "2026-11-04T02:10:42.153Z", "durationMs": 1800415, "success": true, "reason": "tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1788369 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 10976, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": 3, "runtimeFailed": 0 } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 10976 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.1201526975637515 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1788369 }, { "source": "facts", "path": "outcome.durationMs", "value": 1800415 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 300, "peakRssMb": 11010, "peakCpuPercent": 1225.5, "firstAtMs": 1, "lastAtMs": 1783239 }, "gpu": { "available": true, "sampleCount": 300, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 300, "peakUsedMb": 29051, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12830, "totalOutputTokens": 4847, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15368, "todayTokensCached": 14448, "todayTokensOut": 346, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15368, "totalTokensCached": 14448, "totalTokensOut": 346, "totalCost": 0, "medianOutputTokensPerSec": 2.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.9, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7734, "cachedInputTokens": 7114, "outputTokens": 269, "cost": 0, "durationMs": 101072, "outputTokensPerSec": 2.866076184390209, "at": "2026-11-04T01:46:47.095Z" }, "lastUpdated": "2026-11-04T01:46:47.095Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T02:10:35.520Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1787378, "meanTokensPerSec": 2.89, "genTokensPerSec": 2.89, "genTokens": 4847, "promptTokensPerSec": 157.56, "peakRssMb": 11010, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15368, "output": 346 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tank-combat-arcade/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-04T01:44:35.119Z", "completedAt": "2026-11-04T02:10:30.898Z", "ms": 1555779, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "d9fa3f0e-b137-4d98-950b-dbd71ca649ae" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T01:40:47.733Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T01:40:47.616Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1555779 }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "arthur": 2, "torsten": 1 }, "perTask": { "tank-combat-arcade/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 5, "maxContextFill": 0.1066741943359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1066741943359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "durationMs": 1800415, "timeoutMs": 14984391, "budgetUsedFraction": 0.1201526975637515 }, "timing": { "startedAt": "2026-11-04T01:40:41.738Z", "finishedAt": "2026-11-04T02:10:42.153Z", "timeToFirstArtifactMs": 1788369, "timeToLastArtifactWriteMs": 1788453, "timeToFirstTokenMs": 63429, "firstTurnTtftMs": 51939, "timeToFirstToolCallMs": 172123 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 3, "byTool": { "suggest_work": 1, "start_work": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/tank-combat-arcade/index.html", "finalBytes": 10978, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 1790669, "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 10976, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null } ], "latest": { "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 10976, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": 3, "runtimeFailed": 0 } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 1791185, "kind": "structured", "gezel": "torsten", "question": "Your web page is finished. - **`index.html`**", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-04T01:40:41.743Z] [trial] id=tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29 scenario=tankcombat model=underdog-saluki-27b-iq2-mix", "[2026-11-04T01:40:41.743Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "[2026-11-04T01:40:41.743Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-tankcombat-ewfMOs", "[2026-11-04T01:40:41.743Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T01:40:41.743Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T01:40:41.744Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T01:40:41.747Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T01:40:41.754Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-tankcombat-ewfMOs", "[2026-11-04T01:40:41.755Z] [trial] throughput-scaled ceiling: 120m → 250m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T01:40:47.686Z] [trial] daemon spawned pid=50056 port=https://127.0.0.1:57380", "[2026-11-04T01:40:49.079Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T01:40:49.098Z] [trial] meester=torsten", "[2026-11-04T01:40:49.260Z] [trial] sent prompt to meester (1688 chars)", "[2026-11-04T01:40:49.328Z] [poll] initial digests hard=11a7d254fc94212c soft=7ef8e459d3ec39bf", "[2026-11-04T01:43:38.390Z] [poll] hard-progress: 11a7d254fc94212c → f648c4e1241f8b53 after 169s (workspaceProjects=2 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=302,stream=368) sniff=none)", "[2026-11-04T01:44:37.772Z] [poll] hard-progress: f648c4e1241f8b53 → 6e937cfde01b8d8c after 59s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=425,stream=606) sniff=none)", "[2026-11-04T01:45:43.412Z] [poll] hard-progress: 6e937cfde01b8d8c → d2275dec2738d865 after 66s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=589,stream=768) sniff=none)", "[2026-11-04T01:49:41.941Z] [poll] hard-progress: d2275dec2738d865 → 836100a03a401c3b after 239s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=2,slot=1160,stream=1723) sniff=none)", "[2026-11-04T02:10:35.031Z] [poll] terminal: success=true reason=tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "[2026-11-04T02:10:35.582Z] [capture] dumped 3 session(s)", "[2026-11-04T02:10:35.584Z] [capture] snapshotted telemetry for 3 session(s)", "[2026-11-04T02:10:35.635Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T02:10:42.151Z] [trial] daemon shut down", "[2026-11-04T02:10:42.163Z] [trial] PASS duration=1800415ms reason=tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))" ] }, "metrics": { "process": { "sampleCount": 300, "peakRssMb": 11010, "peakCpuPercent": 1225.5, "firstAtMs": 1, "lastAtMs": 1783239 }, "gpu": { "available": true, "sampleCount": 300, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 300, "peakUsedMb": 29051, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12830, "totalOutputTokens": 4847, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15368, "todayTokensCached": 14448, "todayTokensOut": 346, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15368, "totalTokensCached": 14448, "totalTokensOut": 346, "totalCost": 0, "medianOutputTokensPerSec": 2.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.9, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7734, "cachedInputTokens": 7114, "outputTokens": 269, "cost": 0, "durationMs": 101072, "outputTokensPerSec": 2.866076184390209, "at": "2026-11-04T01:46:47.095Z" }, "lastUpdated": "2026-11-04T01:46:47.095Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T02:10:35.520Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1787378, "meanTokensPerSec": 2.89, "genTokensPerSec": 2.89, "genTokens": 4847, "promptTokensPerSec": 157.56, "peakRssMb": 11010, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15368, "output": 346 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T01:40:41.738Z", "finishedAt": "2026-11-04T02:10:42.153Z", "timeToFirstArtifactMs": 1788369, "timeToLastArtifactWriteMs": 1788453, "timeToFirstTokenMs": 63429, "firstTurnTtftMs": 51939, "timeToFirstToolCallMs": 172123 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tank-combat-arcade/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-04T01:44:35.119Z", "completedAt": "2026-11-04T02:10:30.898Z", "ms": 1555779, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "d9fa3f0e-b137-4d98-950b-dbd71ca649ae" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T01:40:47.733Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T01:40:47.616Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1555779 }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "arthur": 2, "torsten": 1 }, "perTask": { "tank-combat-arcade/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 5, "maxContextFill": 0.1066741943359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1066741943359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-tankcombat", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "scenarioId": "tankcombat", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "axisSource": "default HTML axes in evals/src/llm-judge.ts; legacy HTML scenario without overrides", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/workspace/tank-combat-arcade/index.html", "artifactBytes": 10978, "previewPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/judge-preview.png", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 2, "codeQuality": 3, "polish": 3 }, "meanScore": 3.25, "justification": "Readable grid, tanks and HUD, but real Space and WASD do nothing because event.key is matched against event.code names. Arrow keys work. Cooldowns and respawn use millisecond-sized constants with seconds-based dt, so combat delays last minutes. A clean render and input activity pass do not establish a playable game.", "reviewBasis": "Full source below 20 KiB, rendered preview and real keyboard observations saved separately. Model identity was visible in evidence paths." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:56:34.260Z", "reviewId": "003", "cellId": "r1-saluki-cuda-core-tankcombat", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "scenarioId": "tankcombat", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "baselineCellId": "r1-saluki-cuda-core-tankcombat", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b95535e1e21d6351f6f3cbea5653e2f4ff22cb0e52812713de0e203bb3c300bb", "evidenceSha256": "4726ecda49c77f177066d35d232c6a98a20665aa2cde9b7115b4fac34a602eed", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/workspace/tank-combat-arcade/index.html", "bytes": 10978, "sha256": "f82a8b63a40808d8cc75cfca7c4f51e2c1150ecdf5b42521c19fbe19088adc57", "shownBytes": 10978, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 3, "codeQuality": 4, "polish": 3 }, "justification": "The HUD pills, grid canvas, tank sprites and game-over overlay look reasonably tidy. But keyMap is keyed by code names (\"Space\", \"KeyW\", \"KeyR\") while the handlers read ev.key (\" \", \"w\", \"r\"). So Space fire, WASD and R restart never work. The player can't shoot, so the score can't rise. Arrow keys and inputTick do work.", "limitations": "Source-only review; nothing was run. The key-handling bug is inferred from reading the code, not observed at runtime. Visual quality is judged from CSS and canvas draw code only.", "meanScore": 3.75, "durationMs": 18672, "costUsd": 0.0395096, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 7960, "cache_read_input_tokens": 856, "output_tokens": 758, "output_tokens_details": { "thinking_tokens": 413 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 7960, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 758, "cache_read_input_tokens": 856, "cache_creation_input_tokens": 7960, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 7960 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 758, "cacheReadInputTokens": 856, "cacheCreationInputTokens": 7960, "webSearchRequests": 0, "costUSD": 0.0395096, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 413, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 7960, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - tankcombat / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 30m 0s; budget used: 12.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1788369`; `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":10976,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"gam...`; `facts.sniff.latest.bytes: 10976` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.1201526975637515`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 1788369`; `facts.outcome.durationMs: 1800415` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11010 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29051 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.89 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 12830 + 4847 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1556s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 11% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1788369`\n- `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":10976,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"gam...`\n- `facts.sniff.latest.bytes: 10976`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.1201526975637515`\n- `facts.toolUse.totalToolCalls: 3`\n- `facts.timing.timeToFirstArtifactMs: 1788369`\n- `facts.outcome.durationMs: 1800415`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1788369`; `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":10976,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"gam...`; `facts.sniff.latest.bytes: 10976`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.1201526975637515`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 1788369`; `facts.outcome.durationMs: 1800415`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nInline Codex mean **3.25/10**: visualQuality 5, functionalCompleteness 2, codeQuality 3, polish 3. This does not change the fixed **10.0** score above. Full source (10978 bytes), the rendered preview, and real browser keyboard input were inspected. See `judge-preview.png`, `advisory-interaction-observations.json`, and the campaign's `codex-inline-judge.json`.\n\nThe simple grid, tanks, score and lives render cleanly. However, pressing real Space leaves playerBullets and inputTick at zero; W also does nothing. ArrowUp moves the player and increments inputTick. The code indexes `keyMap[ev.key]` using code-style `Space`/`KeyW` keys. R has the same mismatch. Cooldown and respawn constants are decremented by seconds: the player's 180 delay is 180 seconds, enemy fire 900 seconds, and respawn 700 seconds. Thus the displayed game is not a working combat experience despite the narrow keyboard-activity gate passing.\n\n## Positives\n\n- A complete self-contained artifact was written in one write call, with three recorded browser assertions passing (`artifacts.htmlFiles[0]`, `toolUse.byTool.write_file: 1`, `outcome.success: true`, `outcome.reason`). No external assets are required.\n- No malformed-tool red flag or missing role was recorded; one auto-answer kept work moving (`toolUse.redFlags: []`, `team.missingExpectedRoles: []`, `autoAnswer.total: 1`).\n\n## Negatives\n\n- The fixed quality score measures its recorded gate, whose keyboard-activity coverage missed the broken firing controls (`sniff.latest.signals`, `outcome.reason`; separate browser observations above). It must not be interpreted as full gameplay validation.\n- First output arrived at 1788369 ms of a 1800415 ms trial. Sustained decode was 2.89 tokens/sec for 4847 generated tokens (`timing.timeToFirstArtifactMs`, `outcome.durationMs`, `perf.derived`). A high process-efficiency score reflects scaled budget and few calls, not low latency.\n\n## Tactical experiment — not applied\n\nIn the Gilde Builder template, `../gilde/data/gezel-templates/bu/builder/versions/1.5.1/about.md`, extend the existing “Prove it ran” paragraph in a future content version with: “Exercise each advertised control with real user input and verify its effect; check that time-based constants use the same units as elapsed time.” This is a candidate prompt improvement, not a demonstrated fix. The single-write artifact and actual-input observations justify testing it on an unrelated form or interactive dashboard as well as this game. Do not add scenario names or expected selectors to model tuning.\n\n## Strategic experiment — not applied\n\nAdd a reusable optional artifact smoke-check adapter in a proposed `packages/service/src/chat/artifact-smoke-check.ts`: accept user-facing action/expected-effect pairs, execute them through available browser tooling, and return actual event payloads plus observed state to the owner before completion. Keep it separate from frozen evaluation gates. The failure is missing semantic verification, not missing syntax or a render surface; petshop's absent stylesheet independently shows why rendering and asset checks also matter.\n\n## Generalization and next experiment\n\nRun the generic control-verification prompt in a separate A/B campaign on interactive UI work and an unrelated data-entry workflow. Measure working advertised actions and artifact latency without changing the current gates. The current baseline remains untouched. Compared with schema-migration and failing-tests-spec, this trial shows that tool execution and clean compilation/rendering do not establish complete user behavior.\n\n## Last three same-scenario trials\n\n- [tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/facts.json): pass, fixed 10, 51.3 seconds, recorded decode 133.35 tokens/sec.\n- [tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw/facts.json): pass, fixed 10, 56.9 seconds, recorded decode 129.87 tokens/sec.\n- [tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-15-36-256Z-6z5i/facts.json): pass, fixed 10, 57.1 seconds, recorded decode 135.95 tokens/sec.\n\nThese older trials used Gemma4 12B and native 0.1.54. Model, engine and execution conditions differ, so this is historical context, not a controlled speed or quality regression. All three older tankcombat verdicts explicitly lacked browser runtime validation. The new advisory gameplay defects cannot be compared from those fixed scores alone.\n" }, { "cellId": "r1-saluki-cuda-core-schema-migration", "baselineCellId": "r1-saluki-cuda-core-schema-migration", "suite": "core", "scenario": "schema-migration", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T02:13:44.134Z", "finishedAt": "2026-11-04T02:41:12.380Z", "durationMs": 1648246, "success": true, "reason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 220981 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "schema-migration", "bytes": 7171, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.3771344212617153 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 17 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 220981 }, { "source": "facts", "path": "outcome.durationMs", "value": 1648246 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 274, "peakRssMb": 13852, "peakCpuPercent": 1213.3, "firstAtMs": 0, "lastAtMs": 1634418 }, "gpu": { "available": true, "sampleCount": 274, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 274, "peakUsedMb": 31316, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 71556, "totalOutputTokens": 3098, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 84691, "todayTokensCached": 33407, "todayTokensOut": 2759, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 84691, "totalTokensCached": 33407, "totalTokensOut": 2759, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 10 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 14248, "cachedInputTokens": 11948, "outputTokens": 841, "cost": 0, "durationMs": 332695, "outputTokensPerSec": 2.6915248122812843, "at": "2026-11-04T02:41:01.641Z" }, "lastUpdated": "2026-11-04T02:41:01.641Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T02:41:06.832Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1636369, "meanTokensPerSec": 2.79, "genTokensPerSec": 2.79, "genTokens": 3098, "promptTokensPerSec": 159.53, "peakRssMb": 13852, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 84691, "output": 2759 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T02:13:50.074Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T02:13:49.956Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "riley": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 19, "maxContextFill": 0.2023162841796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2023162841796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "durationMs": 1648246, "timeoutMs": 4370447, "budgetUsedFraction": 0.3771344212617153 }, "timing": { "startedAt": "2026-11-04T02:13:44.134Z", "finishedAt": "2026-11-04T02:41:12.380Z", "timeToFirstArtifactMs": 220981, "timeToLastArtifactWriteMs": 1637508, "timeToFirstTokenMs": 46566, "firstTurnTtftMs": 33910, "timeToFirstToolCallMs": 76182 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 17, "byTool": { "read_files": 1, "write_file": 4, "replace_in_file": 7, "read_file": 4, "stat": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 11 }, "sniff": { "progression": [ { "atMs": 9298, "filePath": "schema-migration", "bytes": 1659, "score": 0, "scoreMax": 7, "signals": [], "failReason": "types.ts still declares name or is missing required string firstName/lastName fields on User and CreateUserInput" }, { "atMs": 225257, "filePath": "schema-migration", "bytes": 2729, "score": 0, "scoreMax": 7, "signals": [], "failReason": "types.ts still declares name or is missing required string firstName/lastName fields on User and CreateUserInput" }, { "atMs": 304149, "filePath": "schema-migration", "bytes": 2776, "score": 2, "scoreMax": 7, "signals": [ "types-updated", "migrate-function" ], "failReason": "store.ts still uses a live name property or is missing firstName/lastName" }, { "atMs": 398637, "filePath": "schema-migration", "bytes": 2812, "score": 2, "scoreMax": 7, "signals": [ "types-updated", "migrate-function" ], "failReason": "store.ts still uses a live name property or is missing firstName/lastName" }, { "atMs": 501102, "filePath": "schema-migration", "bytes": 2873, "score": 3, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "migrate-function" ], "failReason": "handlers.ts unmet subcontracts: format-display-full-name (return firstName + space + lastName from formatDisplayName); render-card-full-name (include firstName + space + lastName in the rendered card); summary-full-name (include each mapped user firstName + space + lastName); no-legacy-name-access (remove every live .name property access)" }, { "atMs": 615135, "filePath": "schema-migration", "bytes": 2941, "score": 4, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function" ], "failReason": "tests/migrate.test.ts not present yet" }, { "atMs": 742022, "filePath": "schema-migration", "bytes": 2915, "score": 4, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function" ], "failReason": "tests/migrate.test.ts not present yet" }, { "atMs": 969753, "filePath": "schema-migration", "bytes": 4087, "score": 4, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function" ], "failReason": "tests/migrate.test.ts unmet subcontracts: multi-word-name-case (cover a three-or-more-word name through a full legacy User record) (direct cases=3, valid cases=3, valid legacy-record calls=3, invalid migrateUser calls=0)" }, { "atMs": 1187378, "filePath": "schema-migration", "bytes": 4061, "score": 4, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function" ], "failReason": "tests/migrate.test.ts unmet subcontracts: multi-word-name-case (cover a three-or-more-word name through a full legacy User record) (direct cases=3, valid cases=3, valid legacy-record calls=3, invalid migrateUser calls=0)" }, { "atMs": 1304693, "filePath": "schema-migration", "bytes": 4075, "score": 5, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present" ], "failReason": "MIGRATION.md not present yet" }, { "atMs": 1642198, "filePath": "schema-migration", "bytes": 7171, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } ], "latest": { "filePath": "schema-migration", "bytes": 7171, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T02:13:44.139Z] [trial] id=schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z scenario=schema-migration model=underdog-saluki-27b-iq2-mix", "[2026-11-04T02:13:44.139Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T02:13:44.139Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-schema-migration-ScV4oe", "[2026-11-04T02:13:44.139Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T02:13:44.139Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "[2026-11-04T02:13:44.141Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T02:13:44.143Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T02:13:44.150Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-schema-migration-ScV4oe", "[2026-11-04T02:13:44.150Z] [trial] throughput-scaled ceiling: 35m → 73m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T02:13:50.013Z] [trial] daemon spawned pid=25476 port=https://127.0.0.1:61549", "[2026-11-04T02:13:52.699Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T02:13:52.716Z] [trial] meester=fenton", "[2026-11-04T02:13:52.716Z] [trial] running scenario setup hook", "[2026-11-04T02:13:53.249Z] [trial] scenario setup complete", "[2026-11-04T02:13:53.251Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T02:13:53.377Z] [poll] initial digests hard=d2857336376f244b soft=65a20fa0954dcd82", "[2026-11-04T02:13:53.617Z] [poll] hard-progress: d2857336376f244b → edc9da26dc447944 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/1659B)", "[2026-11-04T02:15:05.157Z] [poll] hard-progress: edc9da26dc447944 → e0db03858068eb36 after 72s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=72,stream=117) sniff=0/1659B)", "[2026-11-04T02:17:29.452Z] [poll] hard-progress: e0db03858068eb36 → 6c8039fb37014973 after 144s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=2,slot=378,stream=717) sniff=0/2729B)", "[2026-11-04T02:18:48.411Z] [poll] hard-progress: 6c8039fb37014973 → 87995350b4e4ad36 after 79s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=3,slot=498,stream=945) sniff=2/2776B)", "[2026-11-04T02:20:22.909Z] [poll] hard-progress: 87995350b4e4ad36 → 1ef76b547b990fd4 after 94s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=4,slot=626,stream=1184) sniff=2/2812B)", "[2026-11-04T02:20:47.362Z] [poll] hard-progress: 1ef76b547b990fd4 → e71371d36ee09ba2 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=5,slot=639,stream=1200) sniff=2/2812B)", "[2026-11-04T02:22:05.364Z] [poll] hard-progress: e71371d36ee09ba2 → 3f59c47b6d181563 after 78s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=5,tools=6,slot=714,stream=1335) sniff=3/2873B)", "[2026-11-04T02:23:59.393Z] [poll] hard-progress: 3f59c47b6d181563 → bee01ecf53dae029 after 114s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=6,tools=7,slot=930,stream=1755) sniff=4/2941B)", "[2026-11-04T02:24:53.526Z] [poll] hard-progress: bee01ecf53dae029 → e9487621ce9bff06 after 54s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=6,tools=8,slot=999,stream=1882) sniff=4/2941B)" ] }, "metrics": { "process": { "sampleCount": 274, "peakRssMb": 13852, "peakCpuPercent": 1213.3, "firstAtMs": 0, "lastAtMs": 1634418 }, "gpu": { "available": true, "sampleCount": 274, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 274, "peakUsedMb": 31316, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 71556, "totalOutputTokens": 3098, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 84691, "todayTokensCached": 33407, "todayTokensOut": 2759, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 84691, "totalTokensCached": 33407, "totalTokensOut": 2759, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 10 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 14248, "cachedInputTokens": 11948, "outputTokens": 841, "cost": 0, "durationMs": 332695, "outputTokensPerSec": 2.6915248122812843, "at": "2026-11-04T02:41:01.641Z" }, "lastUpdated": "2026-11-04T02:41:01.641Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T02:41:06.832Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1636369, "meanTokensPerSec": 2.79, "genTokensPerSec": 2.79, "genTokens": 3098, "promptTokensPerSec": 159.53, "peakRssMb": 13852, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 84691, "output": 2759 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T02:13:44.134Z", "finishedAt": "2026-11-04T02:41:12.380Z", "timeToFirstArtifactMs": 220981, "timeToLastArtifactWriteMs": 1637508, "timeToFirstTokenMs": 46566, "firstTurnTtftMs": 33910, "timeToFirstToolCallMs": 76182 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T02:13:50.074Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T02:13:49.956Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "riley": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 19, "maxContextFill": 0.2023162841796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2023162841796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-schema-migration", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "scenarioId": "schema-migration", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store", "reason": "Scenario has no declared judge axes and no legacy HTML output. Artifacts were read and reviewed in the postmortem; no axis scores invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:56:30.691Z", "reviewId": "004", "cellId": "r1-saluki-cuda-core-schema-migration", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "scenarioId": "schema-migration", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "baselineCellId": "r1-saluki-cuda-core-schema-migration", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "5d4ac2ad3fbaf98e15fdd9b3699a49f91d9a17ff86bb68e6f5e65fd9e1ed745a", "evidenceSha256": "16a597e83116a264dcf2d9b1f6eea59a3b05047f47a9e7350ab4a0cda3e24322", "artifactFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\MIGRATION.md", "bytes": 3110, "sha256": "697739fdfeba053e051f9819ee26ac2a1c9a01582e321f3c9f4d82a377a115f0", "shownBytes": 3110, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\package.json", "bytes": 118, "sha256": "da7cfa9ba7722e4bdeb55cc9b373f3a311dfd6096b48587df9fe10286cd581fc", "shownBytes": 118, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\README.md", "bytes": 233, "sha256": "e8ad3a935d40b5659206cfec22cc684ef10b2e5e673130d9e501d192d87d1194", "shownBytes": 233, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\src\\handlers.ts", "bytes": 614, "sha256": "f151082616220d177a6e1bc1d525727b820376df24996db8e901924a3d2b1b85", "shownBytes": 614, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\src\\migrate.ts", "bytes": 1018, "sha256": "e9f88335fc8ef947281f981749163f56d5480c61adb692433e3790de30a346d2", "shownBytes": 1018, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\src\\store.ts", "bytes": 835, "sha256": "6c877c17a044a87df7ed7af1488eb1f879d841902fd183eba20260a8fe34c735", "shownBytes": 835, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\src\\types.ts", "bytes": 422, "sha256": "64ba55edba66e2e1b6867c13b0c25aed688e080b5546141a031606743070b694", "shownBytes": 422, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\tests\\migrate.test.ts", "bytes": 1186, "sha256": "69aeada66d159a38bf67ea3d7129c8201394d56938c1a61a84a727edce09e818", "shownBytes": 1186, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z\\workspace\\user-store\\tsconfig.json", "bytes": 323, "sha256": "4d432e97e14419c8cd5438b66f9177e84275f76e029c7f13d30683d65fec7c38", "shownBytes": 323, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "All six targets exist. types, store and handlers use firstName/lastName. migrateUser splits on the first space and spreads the rest, preserving other fields. Three tests cover normal, one-word and multi-word names. MIGRATION.md is 3110 bytes. tsconfig.json and package.json look untouched. One unsupported claim: MIGRATION.md says typecheck passes.", "limitations": "Source-only review. Nothing was executed, so the typecheck and vitest results are unverified. tsconfig excludes tests, so test types are checked elsewhere. The \"typecheck passes\" claim in MIGRATION.md is unsupported. LegacyUser is typed narrowly, so extra fields would not typecheck. Comments in store.ts and handlers.ts are stale.", "meanScore": null, "durationMs": 15090, "costUsd": 0.030564, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6705, "cache_read_input_tokens": 0, "output_tokens": 374, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6705, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 374, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 6705, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6705 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 374, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 6705, "webSearchRequests": 0, "costUSD": 0.030564, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6705, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - schema-migration / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 27m 28s; budget used: 37.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 220981`; `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":7171,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.3771344212617153`; `facts.toolUse.totalToolCalls: 17`; `facts.timing.timeToFirstArtifactMs: 220981`; `facts.outcome.durationMs: 1648246` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13852 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31316 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.79 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 71556 + 3098 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 20% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 220981`\n- `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":7171,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.3771344212617153`\n- `facts.toolUse.totalToolCalls: 17`\n- `facts.timing.timeToFirstArtifactMs: 220981`\n- `facts.outcome.durationMs: 1648246`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 220981`; `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":7171,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.3771344212617153`; `facts.toolUse.totalToolCalls: 17`; `facts.timing.timeToFirstArtifactMs: 220981`; `facts.outcome.durationMs: 1648246`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nNo `judge.axes` declaration exists in this scenario, and it has no HTML artifact. Recorded as `notApplicable` in the inline-review ledger without invented axis scores; output is present, so this is not `noArtifact`.\n\nRead the produced types, store, handlers, migration function, tests and MIGRATION.md. The implementation removes name from both input/domain types, updates consumers, splits at the first space, and preserves non-name properties by spreading the rest of the object. Three authored tests cover two-, one-, and multi-word names. The document explains the API break and sorting behavior. Stale comments in store/handlers still describe the old name field. The explicit MigratedUser return type does not expose additional preserved fields to callers; no requirement-specific failure is inferred from that limitation.\n\n## Positives\n\n- All seven acceptance signals passed, including the compiler gate and migration documentation (`sniff.latest.score: 7`, `scoreMax: 7`, `signals`, `outcome.success: true`). The code and explanatory document agree on the primary schema change.\n- The developer used focused edits and a batch read without red flags or intervention (`toolUse.byTool.replace_in_file: 7`, `read_files: 1`, `redFlags: []`, `autoAnswer.total: 0`).\n\n## Negatives\n\n- Seventeen calls and 27m28s were required, at 2.79 generated tokens/sec and 159.53 prompt tokens/sec (`toolUse.totalToolCalls`, `outcome.durationMs: 1648246`, `perf.derived`). Success is clear; interactive responsiveness is limited.\n- Process RSS peaked at 13852 MiB and total system use at 31316/31865 MiB (`perf.process.peakRssMb`, `perf.systemMemory`). This machine has little observed total-RAM headroom even though VRAM stayed within capacity. Whole-system memory includes other applications.\n\n## Tactical experiment — not applied\n\nFor a later isolated A/B, change `tuning.reasoning.thinkingBudget` and the selected thinking-general/thinking-coding profile override from 4096 to 2048 in `../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json`. This is a cost-cap hypothesis motivated by high end-to-end latency across this trial and tankcombat; it is not evidence that hidden reasoning alone caused the delay. Preserve output-token caps and run both this migration and an unrelated productivity task. Reject if repair success or completeness drops.\n\n## Strategic experiment — not applied\n\nExtend `packages/inference/src/providers/llama-cpp/offload-planner.ts` and `stdout-parser.ts` with a stable structured record pairing chosen CPU FFN layers, KV reservation, actual context, and each turn's prompt/decode time. Aggregate it per trial in a proposed `evals/src/offload-diagnostics.ts`. The current facts show similar 2.8 tokens/sec across three different workloads and logs show 25 CPU FFN layers; structured correlation would make future planner changes measurable without attributing slowdown to the model or GPU capacity alone.\n\n## Generalization and next experiment\n\nKeep this baseline fixed. In a later A/B, test the smaller reasoning cap on schema migration and an unrelated productivity artifact, comparing acceptance, first-artifact time, repair turns and total tokens. Failing-tests-spec independently demonstrates productive multi-step repair, while tankcombat demonstrates that a passing surface check can hide semantic errors. Any broad performance improvement must preserve both code correctness and artifact behavior.\n\n## Last three same-scenario trials\n\n- [schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/facts.json): pass, fixed 9, 209.6 seconds, recorded decode 118.49 tokens/sec.\n- [schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7/facts.json): pass, fixed 9.5, 114.2 seconds, recorded decode 123.62 tokens/sec.\n- [schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-18-27-491Z-twse/facts.json): pass, fixed 9.5, 162.7 seconds, recorded decode 126.21 tokens/sec.\n\nThese older trials used Gemma4 12B and native 0.1.54. Model, engine and execution conditions differ, so this is historical context, not a controlled speed or quality regression. All three older migration runs passed. The current fixed score remains within their 9.0–9.5 range; latency is much higher and must be reported separately.\n" }, { "cellId": "r1-saluki-cuda-core-failing-tests-spec", "baselineCellId": "r1-saluki-cuda-core-failing-tests-spec", "suite": "core", "scenario": "failing-tests-spec", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T02:44:03.486Z", "finishedAt": "2026-11-04T03:03:06.590Z", "durationMs": 1143104, "success": true, "reason": "vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 262797 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "failing-tests-spec", "bytes": 1494, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.36617433566066193 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 16 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 262797 }, { "source": "facts", "path": "outcome.durationMs", "value": 1143104 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 190, "peakRssMb": 13742, "peakCpuPercent": 1208.7, "firstAtMs": 1, "lastAtMs": 1126417 }, "gpu": { "available": true, "sampleCount": 190, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 190, "peakUsedMb": 30727, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 80496, "totalOutputTokens": 1469, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 63459, "todayTokensCached": 27170, "todayTokensOut": 1157, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 63459, "totalTokensCached": 27170, "totalTokensOut": 1157, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 20342, "cachedInputTokens": 12861, "outputTokens": 115, "cost": 0, "durationMs": 215979, "outputTokensPerSec": 2.6484511462752116, "at": "2026-11-04T03:02:52.674Z" }, "lastUpdated": "2026-11-04T03:02:52.674Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T03:02:58.811Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1129053, "meanTokensPerSec": 2.86, "genTokensPerSec": 2.86, "genTokens": 1469, "promptTokensPerSec": 146.9, "peakRssMb": 13742, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 63459, "output": 1157 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T02:44:09.081Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T02:44:08.973Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "priya": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 10, "maxContextFill": 0.2294921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2294921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "durationMs": 1143104, "timeoutMs": 3121748, "budgetUsedFraction": 0.36617433566066193 }, "timing": { "startedAt": "2026-11-04T02:44:03.486Z", "finishedAt": "2026-11-04T03:03:06.590Z", "timeToFirstArtifactMs": 262797, "timeToLastArtifactWriteMs": 1129159, "timeToFirstTokenMs": 46778, "firstTurnTtftMs": 35472, "timeToFirstToolCallMs": 71992 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 16, "byTool": { "read_file": 11, "write_file": 1, "replace_in_file": 4 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 10797, "filePath": "failing-tests-spec", "bytes": 402, "score": 2, "scoreMax": 4, "signals": [ "tests-untouched", "tsc-clean" ], "failReason": "vitest run failed (exit 1): × starts in \"draft\" 4ms | × starts with an empty history log 0ms | × submit moves draft to pending and returns the new state 0ms | × pay moves pending to paid 0ms | FAIL tests/machine.test.ts > order machine: initial state > starts in \"draft\" | Error: createOrderMachine is not implemented yet | FAIL tests/machine.test.ts > order machine: initial state > starts with an empty history log | FAIL tests/machine.test.ts > order machine: transitions > submit moves draft to pending and returns the new state" }, { "atMs": 264316, "filePath": "failing-tests-spec", "bytes": 402, "score": 2, "scoreMax": 4, "signals": [ "tests-untouched", "vitest-green" ], "failReason": "tsc --noEmit failed (exit 2): src/machine.ts(17,3): error TS2739: Type '{ submit: string; cancel: string; }' is missing the following properties from type 'Record': pay, ship, deliver, expedite" }, { "atMs": 269424, "filePath": "failing-tests-spec", "bytes": 1484, "score": 3, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green" ], "failReason": "tsc --noEmit failed (exit 2): src/machine.ts(17,3): error TS2739: Type '{ submit: string; cancel: string; }' is missing the following properties from type 'Record': pay, ship, deliver, expedite" }, { "atMs": 460843, "filePath": "failing-tests-spec", "bytes": 1414, "score": 2, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented" ], "failReason": "vitest run failed (exit 1): × submit moves draft to pending and returns the new state 4ms | × a rejected send leaves the state unchanged 1ms | FAIL tests/machine.test.ts > order machine: transitions > submit moves draft to pending and returns the new state | AssertionError: expected 'draft' to be 'pending' // Object.is equality | FAIL tests/machine.test.ts > order machine: invalid transitions > a rejected send leaves the state unchanged || tsc --noEmit failed (exit 2): src/machine.ts(17,3): error TS2739: Type '{ submit: string; cancel: string; }' is missing the following properties from type 'Record': pay, ship, deliver, expedite" }, { "atMs": 685377, "filePath": "failing-tests-spec", "bytes": 1423, "score": 2, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented" ], "failReason": "vitest run failed (exit 1): × submit moves draft to pending and returns the new state 6ms | × a rejected send leaves the state unchanged 1ms | FAIL tests/machine.test.ts > order machine: transitions > submit moves draft to pending and returns the new state | AssertionError: expected 'draft' to be 'pending' // Object.is equality | FAIL tests/machine.test.ts > order machine: invalid transitions > a rejected send leaves the state unchanged || tsc --noEmit failed (exit 2): src/machine.ts(42,33): error TS2322: Type 'string | undefined' is not assignable to type 'string'." }, { "atMs": 913070, "filePath": "failing-tests-spec", "bytes": 1424, "score": 3, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "tsc-clean" ], "failReason": "vitest run failed (exit 1): × submit moves draft to pending and returns the new state 5ms | × a rejected send leaves the state unchanged 1ms | FAIL tests/machine.test.ts > order machine: transitions > submit moves draft to pending and returns the new state | AssertionError: expected 'draft' to be 'pending' // Object.is equality | FAIL tests/machine.test.ts > order machine: invalid transitions > a rejected send leaves the state unchanged" }, { "atMs": 1134865, "filePath": "failing-tests-spec", "bytes": 1494, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } ], "latest": { "filePath": "failing-tests-spec", "bytes": 1494, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T02:44:03.492Z] [trial] id=failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo scenario=failing-tests-spec model=underdog-saluki-27b-iq2-mix", "[2026-11-04T02:44:03.492Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T02:44:03.492Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-failing-tests-spec-TkA93i", "[2026-11-04T02:44:03.492Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "[2026-11-04T02:44:03.492Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T02:44:03.494Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T02:44:03.496Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T02:44:03.504Z] [trial] throughput-scaled ceiling: 25m → 52m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T02:44:03.504Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-failing-tests-spec-TkA93i", "[2026-11-04T02:44:08.998Z] [trial] daemon spawned pid=30008 port=https://127.0.0.1:57797", "[2026-11-04T02:44:10.655Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T02:44:10.677Z] [trial] meester=anika", "[2026-11-04T02:44:10.678Z] [trial] running scenario setup hook", "[2026-11-04T02:44:11.162Z] [trial] scenario setup complete", "[2026-11-04T02:44:11.163Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T02:44:11.257Z] [poll] initial digests hard=4ebee2c274fb727a soft=f0779b44f7cb7d9a", "[2026-11-04T02:44:14.338Z] [poll] hard-progress: 4ebee2c274fb727a → c526bf3be2be5d9b after 3s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=2/402B)", "[2026-11-04T02:45:16.837Z] [poll] hard-progress: c526bf3be2be5d9b → a2a898082f41c81d after 62s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=33,stream=48) sniff=2/402B)", "[2026-11-04T02:48:27.873Z] [poll] hard-progress: a2a898082f41c81d → e1b7ab07dead0b58 after 191s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=4,slot=437,stream=842) sniff=2/402B)", "[2026-11-04T02:48:32.987Z] [poll] hard-progress: e1b7ab07dead0b58 → ca0e85ccf69777ce after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=4,slot=438,stream=842) sniff=3/1484B)", "[2026-11-04T02:49:14.027Z] [poll] hard-progress: ca0e85ccf69777ce → fb878b92835595ad after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=5,slot=454,stream=861) sniff=3/1484B)", "[2026-11-04T02:50:07.865Z] [poll] hard-progress: fb878b92835595ad → ece1aa923bb990fd after 54s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=6,slot=471,stream=882) sniff=3/1484B)", "[2026-11-04T02:51:44.466Z] [poll] hard-progress: ece1aa923bb990fd → 8bd6294e8dbdf589 after 97s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=7,slot=543,stream=1009) sniff=2/1414B)", "[2026-11-04T02:52:24.530Z] [poll] hard-progress: 8bd6294e8dbdf589 → 8a908e5c4a3b300a after 40s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=8,slot=559,stream=1027) sniff=2/1414B)", "[2026-11-04T02:53:11.951Z] [poll] hard-progress: 8a908e5c4a3b300a → 54cfe51235cc7c8d after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=9,slot=575,stream=1048) sniff=2/1414B)" ] }, "metrics": { "process": { "sampleCount": 190, "peakRssMb": 13742, "peakCpuPercent": 1208.7, "firstAtMs": 1, "lastAtMs": 1126417 }, "gpu": { "available": true, "sampleCount": 190, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 190, "peakUsedMb": 30727, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 80496, "totalOutputTokens": 1469, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 63459, "todayTokensCached": 27170, "todayTokensOut": 1157, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 63459, "totalTokensCached": 27170, "totalTokensOut": 1157, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 20342, "cachedInputTokens": 12861, "outputTokens": 115, "cost": 0, "durationMs": 215979, "outputTokensPerSec": 2.6484511462752116, "at": "2026-11-04T03:02:52.674Z" }, "lastUpdated": "2026-11-04T03:02:52.674Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T03:02:58.811Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1129053, "meanTokensPerSec": 2.86, "genTokensPerSec": 2.86, "genTokens": 1469, "promptTokensPerSec": 146.9, "peakRssMb": 13742, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 63459, "output": 1157 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T02:44:03.486Z", "finishedAt": "2026-11-04T03:03:06.590Z", "timeToFirstArtifactMs": 262797, "timeToLastArtifactWriteMs": 1129159, "timeToFirstTokenMs": 46778, "firstTurnTtftMs": 35472, "timeToFirstToolCallMs": 71992 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T02:44:09.081Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T02:44:08.973Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "priya": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 10, "maxContextFill": 0.2294921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2294921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-failing-tests-spec", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "scenarioId": "failing-tests-spec", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/workspace/machine-kata", "reason": "Scenario has no declared judge axes and no legacy HTML output. Artifacts were read and reviewed in the postmortem; no axis scores invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:56:45.157Z", "reviewId": "005", "cellId": "r1-saluki-cuda-core-failing-tests-spec", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "scenarioId": "failing-tests-spec", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "baselineCellId": "r1-saluki-cuda-core-failing-tests-spec", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b5eaff4b123cbda2ed1164cf3b48c7bce72bb4c440ef1424080d00167b5db433", "evidenceSha256": "e62a7e7cb191d52fb800d3134dbc26ba556960f6fe24a5480eb6f77a58e17aff", "artifactFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo\\workspace\\machine-kata\\package.json", "bytes": 205, "sha256": "6f3c6e522efed23d503e1270df18e6aef4f87a411c7be2d78cdb24842feb392d", "shownBytes": 205, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo\\workspace\\machine-kata\\src\\machine.ts", "bytes": 1494, "sha256": "126ab4f1e41d0ff81d190c11370c35f30a30cea15832196b08012d359e3c764f", "shownBytes": 1494, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo\\workspace\\machine-kata\\tests\\machine.test.ts", "bytes": 4926, "sha256": "12e2d36908178da95c12c9d5bf5d3bb7d8c67606f2ec4faa3ffd5fff62517997", "shownBytes": 4926, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo\\workspace\\machine-kata\\tsconfig.json", "bytes": 266, "sha256": "8bda3c0260964f6b5352b61969a681b7760822e954787a150c0e959839d389e3", "shownBytes": 266, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "Reading the shown source, src/machine.ts appears to satisfy every assertion. It has the draft/pending/paid/shipped/delivered/cancelled transitions, a guarded expedite, can() that never throws, a history log, and the exact error message format. Unknown events are rejected. The test file looks unmodified.", "limitations": "Source-only review; no tests or tsc were run. Whether the tests and strict tsc actually passed is unobserved. The unmodified tests file is inferred from its content only, since no original hash was provided. There was no transcript of the runner's process.", "meanScore": null, "durationMs": 14452, "costUsd": 0.023382, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5067, "cache_read_input_tokens": 700, "output_tokens": 304, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5067, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 304, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 5067, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5067 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 304, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 5067, "webSearchRequests": 0, "costUSD": 0.023382, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5067, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - failing-tests-spec / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 19m 3s; budget used: 36.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 262797`; `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1494,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.36617433566066193`; `facts.toolUse.totalToolCalls: 16`; `facts.timing.timeToFirstArtifactMs: 262797`; `facts.outcome.durationMs: 1143104` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13742 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10632 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30727 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.86 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 80496 + 1469 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 23% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 262797`\n- `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1494,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.36617433566066193`\n- `facts.toolUse.totalToolCalls: 16`\n- `facts.timing.timeToFirstArtifactMs: 262797`\n- `facts.outcome.durationMs: 1143104`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 262797`; `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1494,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.36617433566066193`; `facts.toolUse.totalToolCalls: 16`; `facts.timing.timeToFirstArtifactMs: 262797`; `facts.outcome.durationMs: 1143104`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nThe scenario declares no `judge.axes` and produces TypeScript rather than HTML. Record `notApplicable` in the inline-review ledger, without inventing qualitative scores or marking the present artifact absent.\n\nRead src/machine.ts and the unchanged 17-test spec. The delivered table-driven machine implements state transitions, guarded expedite, history and rejection behavior. A remaining source-level edge case lies outside the seeded tests: `event in allowed` includes inherited Object properties, so arbitrary strings such as toString can be accepted. Treat this as an advisory robustness finding, not a retroactive gate failure. The artifact was not edited.\n\n## Positives\n\n- The seeded spec remained untouched and both Vitest and strict TypeScript checks passed (`sniff.latest.signals: tests-untouched, machine-implemented, vitest-green, tsc-clean`; `outcome.success: true`).\n- Bounded feedback led to correction of compiler errors and stale returned state: the log moves from TS2739/TS2322 and failed submit/rejected-send cases to green (`sniff.latest.score: 4/4`, `toolUse.byTool.replace_in_file: 4`; corroborated by log.txt at 02:48–03:02Z). No intervention, missing role or red flag was recorded (`autoAnswer.total: 0`, `team.missingExpectedRoles: []`, `toolUse.redFlags: []`).\n\n## Negatives\n\n- Eleven individual reads and four repair edits contributed to a 19m3s completion time (`toolUse.byTool.read_file: 11`, `replace_in_file: 4`, `outcome.durationMs: 1143104`). At 2.86 decode and 146.90 prompt tokens/sec, repeated review is expensive (`perf.derived`). The read count alone does not establish that every reread was redundant.\n- The final gate establishes the seeded contract, not exhaustive robustness; the inherited-property issue above remains in the five-file workspace (`artifacts.otherFileCount: 5`, `sniff.latest.signals`; source-level advisory evidence in src/machine.ts).\n\n## Tactical experiment — not applied\n\nIn a new version of `../gilde/data/gezel-templates/de/developer/versions/1.5.1/about.md`, append to “After a failing check”: “After a type repair, rerun the behavioral checks; after a behavior repair, rerun type checking. Verify getters and returned snapshots reflect later mutations.” This makes the existing re-verification rule concrete for the observed compiler/behavior alternation. Test on another stateful workflow plus a data transformation; do not insert the seeded spec's expected strings or transitions.\n\n## Strategic experiment — not applied\n\nIn `evals/src/sniff-feedback.ts`, prototype a revision-stamped combined feedback record listing all currently failing contracts and the workspace revision each result validated. Preserve current assertions, retry budget and deadlines. A repair should not mistake an older compiler or behavioral result for the current revision. The recorded alternating Vitest/tsc failures justify measuring fewer repair turns; they do not prove stale feedback was their cause. Generalize with schema migration and unrelated artifact-validation scenarios in a separate baseline.\n\n## Generalization and next experiment\n\nCompare the current Developer wording with the generic re-verification addition in a separate A/B across this task and a data-processing task. Measure first-pass dual-gate success, total reads, edits and duration. Do not weaken read-only specs or retry capability failures. Unlike the HTML game's shallow pass, this trial exercised the complete seeded behavior under a real test runner, while still leaving ordinary out-of-spec robustness limits.\n\n## Last three same-scenario trials\n\n- [failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/facts.json): fail, fixed 6.1, 136.8 seconds, recorded decode 148.97 tokens/sec.\n- [failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t/facts.json): pass, fixed 10, 30.0 seconds, recorded decode 139.69 tokens/sec.\n- [failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-26-39-659Z-1pvn/facts.json): pass, fixed 10, 31.1 seconds, recorded decode 136.21 tokens/sec.\n\nThese older trials used Gemma4 12B and native 0.1.54. Model, engine and execution conditions differ, so this is historical context, not a controlled speed or quality regression. The latest older run exhausted repair with TS2322; the two preceding runs passed. The current run completed both gates, but one successful trial does not establish a reliability advantage.\n" }, { "cellId": "r1-saluki-cuda-core-symptom-debug", "baselineCellId": "r1-saluki-cuda-core-symptom-debug", "suite": "core", "scenario": "symptom-debug", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T03:05:54.028Z", "finishedAt": "2026-11-04T03:13:37.072Z", "durationMs": 463044, "success": true, "reason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 455120 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "symptom-debug", "bytes": 1222, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.18541050108532917 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 455120 }, { "source": "facts", "path": "outcome.durationMs", "value": 463044 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 77, "peakRssMb": 9455, "peakCpuPercent": 1204.5, "firstAtMs": 0, "lastAtMs": 450878 }, "gpu": { "available": true, "sampleCount": 77, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 77, "peakUsedMb": 26869, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 5344, "totalOutputTokens": 1151, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T03:13:32.755Z" } }, "source": "engine-log" }, "derived": { "durationMs": 452492, "meanTokensPerSec": 2.88, "genTokensPerSec": 2.88, "genTokens": 1151, "promptTokensPerSec": 163.88, "peakRssMb": 9455, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T03:05:59.700Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T03:05:59.586Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "theo": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0552825927734375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0552825927734375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "durationMs": 463044, "timeoutMs": 2497399, "budgetUsedFraction": 0.18541050108532917 }, "timing": { "startedAt": "2026-11-04T03:05:54.028Z", "finishedAt": "2026-11-04T03:13:37.072Z", "timeToFirstArtifactMs": 455120, "timeToLastArtifactWriteMs": 455120, "timeToFirstTokenMs": 45050, "firstTurnTtftMs": 33866, "timeToFirstToolCallMs": 70446 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 3, "byTool": { "read_file": 2, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 7933, "filePath": "symptom-debug", "bytes": 1193, "score": 1, "scoreMax": 3, "signals": [ "accept-untouched" ], "failReason": "node accept.mjs still failing: CASE 3: expected \"a4\" got \"a5\" | CASE 4: expected [\"a1\",\"a2\",\"a3\",\"a4\",\"a5\",\"a6\",\"a7\",\"a8\",\"a9\",\"a10\"] got [\"a1\",\"a2\",\"a3\",\"a5\",\"a6\",\"a7\",\"a9\",\"a10\"] | CASE 6: expected [\"a1\",\"a2\",\"a3\",\"a4\",\"a5\",\"a6\",\"a7\",\"a8\",\"a9\",\"a10\"] got [\"a1\",\"a2\",\"a3\",\"a4\",\"a6\",\"a7\",\"a8\",\"a9\"]" }, { "atMs": 458470, "filePath": "symptom-debug", "bytes": 1222, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } ], "latest": { "filePath": "symptom-debug", "bytes": 1222, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T03:05:54.032Z] [trial] id=symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1 scenario=symptom-debug model=underdog-saluki-27b-iq2-mix", "[2026-11-04T03:05:54.033Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T03:05:54.032Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "[2026-11-04T03:05:54.032Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-symptom-debug-GtRMIs", "[2026-11-04T03:05:54.033Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T03:05:54.034Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T03:05:54.037Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T03:05:54.044Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-symptom-debug-GtRMIs", "[2026-11-04T03:05:54.044Z] [trial] throughput-scaled ceiling: 20m → 42m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T03:05:59.687Z] [trial] daemon spawned pid=44280 port=https://127.0.0.1:57537", "[2026-11-04T03:06:01.235Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T03:06:01.253Z] [trial] meester=sulien", "[2026-11-04T03:06:01.253Z] [trial] running scenario setup hook", "[2026-11-04T03:06:01.718Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T03:06:01.716Z] [trial] scenario setup complete", "[2026-11-04T03:06:01.797Z] [poll] initial digests hard=b9ce474d4fd094bb soft=6ae00b7d1178f200", "[2026-11-04T03:06:02.165Z] [poll] hard-progress: b9ce474d4fd094bb → 899706d4db06161c after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/1193B)", "[2026-11-04T03:07:07.918Z] [poll] hard-progress: 899706d4db06161c → 486f50770a0a030b after 66s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=42,stream=51) sniff=1/1193B)", "[2026-11-04T03:13:32.498Z] [poll] terminal: success=true reason=node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "[2026-11-04T03:13:32.782Z] [capture] dumped 1 session(s)", "[2026-11-04T03:13:32.786Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T03:13:32.850Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T03:13:37.071Z] [trial] daemon shut down", "[2026-11-04T03:13:37.081Z] [trial] PASS duration=463044ms reason=node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)" ] }, "metrics": { "process": { "sampleCount": 77, "peakRssMb": 9455, "peakCpuPercent": 1204.5, "firstAtMs": 0, "lastAtMs": 450878 }, "gpu": { "available": true, "sampleCount": 77, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 77, "peakUsedMb": 26869, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 5344, "totalOutputTokens": 1151, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T03:13:32.755Z" } }, "source": "engine-log" }, "derived": { "durationMs": 452492, "meanTokensPerSec": 2.88, "genTokensPerSec": 2.88, "genTokens": 1151, "promptTokensPerSec": 163.88, "peakRssMb": 9455, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T03:05:54.028Z", "finishedAt": "2026-11-04T03:13:37.072Z", "timeToFirstArtifactMs": 455120, "timeToLastArtifactWriteMs": 455120, "timeToFirstTokenMs": 45050, "firstTurnTtftMs": 33866, "timeToFirstToolCallMs": 70446 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T03:05:59.700Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T03:05:59.586Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "theo": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0552825927734375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0552825927734375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-symptom-debug", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "scenarioId": "symptom-debug", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/workspace/pagination-service/lib/paginate.mjs", "artifactBytes": 1222, "reason": "Scenario has no declared judge axes and no legacy HTML output. Source and acceptance script reviewed in postmortem; no scores invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:56:47.622Z", "reviewId": "006", "cellId": "r1-saluki-cuda-core-symptom-debug", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "scenarioId": "symptom-debug", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "baselineCellId": "r1-saluki-cuda-core-symptom-debug", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "726851a23eb5f50ccb4b04e3b6fd21b74d3bfebde7484b0b00d5ca15978e21f4", "evidenceSha256": "3a47336444ef429637ee70e4448fe2305c05db14dcd5c9b476f53cd28bed496e", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/workspace/pagination-service/lib/paginate.mjs", "bytes": 1222, "sha256": "e534c01618f7c0906a7b865e21cf9bbee21e313eaaa8fb02fdd14a2b078f3353", "shownBytes": 1222, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes are declared. The shown paginate.mjs looks internally consistent: the cursor stores the next index, and slice/nextIndex handle it correctly. But accept.mjs, its output and the original file are missing, so I can't tell whether the defect was fixed or whether accept.mjs was left untouched.", "limitations": "Only the final paginate.mjs is shown. There is no accept.mjs, no CASE output, no diff or original source, and no run transcript. Passing and the untouched accept.mjs are unverified. Source-only review.", "meanScore": null, "durationMs": 14835, "costUsd": 0.011592, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2137, "cache_read_input_tokens": 700, "output_tokens": 297, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2137, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 297, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2137, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2137 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 297, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2137, "webSearchRequests": 0, "costUSD": 0.011592, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2137, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - symptom-debug / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 7m 43s; budget used: 18.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 455120`; `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1222,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.18541050108532917`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 455120`; `facts.outcome.durationMs: 463044` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9455 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26869 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.88 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 5344 + 1151 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 455120`\n- `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1222,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.18541050108532917`\n- `facts.toolUse.totalToolCalls: 3`\n- `facts.timing.timeToFirstArtifactMs: 455120`\n- `facts.outcome.durationMs: 463044`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 455120`; `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1222,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.18541050108532917`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 455120`; `facts.outcome.durationMs: 463044`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nThis scenario declares no judge axes and produces JavaScript rather than a legacy HTML artifact. Its review ledger entry is notApplicable; output is present, so noArtifact would be incorrect. No qualitative axis scores were invented.\n\nRead the complete 1222-byte lib/paginate.mjs and the untouched acceptance script. The change removes the extra increment when decoding the next-page cursor and updates the adjacent comment to match. It preserves validation, slicing and end-of-list behavior. The eight acceptance cases cover page boundaries, complete traversal with two limits, uniqueness, oversized pages and an empty list. The harness ran the real acceptance script and observed ALL CASES PASS.\n\n## Positives\n\n- One targeted edit after two reads corrected the semantic defect (toolUse.byTool: read_file 2, replace_in_file 1; sniff.latest.score: 3/3). The acceptance script stayed untouched (sniff.latest.signals: accept-untouched, paginate-changed, accept-passes).\n- No extra intervention, role gap or tool red flag was recorded (autoAnswer.total: 0; team.missingExpectedRoles: []; toolUse.redFlags: []). The source change is small and directly explains the original missing-item failures.\n\n## Negatives\n\n- Even this small repair took 463044 ms, or 7m43s, with the first artifact at 455120 ms (outcome.durationMs; timing.timeToFirstArtifactMs). A fixed 10.0 measures successful delivery within the scaled budget, not interactive speed.\n- Decode averaged 2.88 tokens/sec over 1151 generated tokens, while prompt processing averaged 163.88 tokens/sec (perf.derived). Two reads and a small replacement had negligible tool execution duration in recording/transcript.json; inference dominated the observed wait. The evidence does not isolate all generation into hidden reasoning.\n\n## Memory and execution\n\nPeak VRAM was 10626/12227 MiB, process RSS 9455 MiB, and whole-system memory 26869/31865 MiB (perf.gpu, perf.process, perf.systemMemory). Launch telemetry records native 0.1.57, CUDA build 11960, one 65536-token slot and f16 KV. The offload plan keeps dense FFN for 39/64 layers on GPU and 25 on CPU. Successful execution is mixed CPU/GPU operation; shared GPU memory is not measured separately. No explicit OOM, context-overflow or unexpected engine exit was found.\n\n## Tactical experiment — not applied\n\nIn a separate A/B, reduce tuning.reasoning.thinkingBudget and the selected thinking-general/thinking-coding profile override from 4096 to 512 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json. Keep output caps and sampling unchanged. This is a deliberately bounded reasoning-cost experiment for short repairs, motivated by 1151 generated tokens and long first-artifact latency; it is not a proven fix and may harm harder work. Measure first-artifact time, repair success and total tokens on this task plus failing-tests-spec and an unrelated productivity task. Do not change the current campaign.\n\n## Strategic experiment — not applied\n\nExtend packages/inference/src/providers/llama-cpp/offload-planner.ts and stdout-parser.ts to emit a stable record linking CPU FFN count, KV reservation and context to measured per-turn prompt/decode time. A proposed evals/src/offload-diagnostics.ts can aggregate these records for later comparisons of the existing f16-first policy with alternative memory plans. CPU FFN placement is explicit, and approximately 2.8 tokens/sec now recurs across migration, state-machine repair and pagination. That is sufficient to investigate the hosting plan, but does not prove CPU offload alone explains the slowdown. Keep capability outcomes and performance measurements separate.\n\n## Generalization and next experiment\n\nThe general behavior demonstrated here is minimal semantic repair from concrete diagnostics while preserving read-only acceptance checks. Compare with failing-tests-spec, which also passed but needed four edits, and tankcombat, whose shallow gate missed broken controls. A smaller reasoning cap should be tested against both successful repairs and unrelated tasks before being recommended. No scenario-specific hints, selectors or expected answers belong in the model profile.\n\n## Last three same-scenario trials\n\n- [symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/facts.json): pass, fixed 10, 35.5 seconds, 7 tool calls, recorded decode 118.38 tokens/sec.\n- [symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283/facts.json): pass, fixed 10, 50.8 seconds, 7 tool calls, recorded decode 129.73 tokens/sec.\n- [symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-03-382Z-96bu/facts.json): pass, fixed 10, 43.3 seconds, 4 tool calls, recorded decode 117.52 tokens/sec.\n\nAll three older trials used Gemma4 12B and native 0.1.54. They also passed the acceptance gate. The new run retains fixed score 10.0 with fewer tool calls, but much higher latency. Different model, engine and execution conditions prevent a controlled regression claim. This single first-round result does not establish three-trial reliability.\n" }, { "cellId": "r1-saluki-cuda-core-data-wrangle", "baselineCellId": "r1-saluki-cuda-core-data-wrangle", "suite": "core", "scenario": "data-wrangle", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T03:15:59.498Z", "finishedAt": "2026-11-04T04:12:22.277Z", "durationMs": 3382779, "success": false, "reason": "no real progress for 1251s — model busy but not delivering (hard digest stuck at 2621efcc102c5c56)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "data-wrangle", "score": 0, "bytes": 0 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1251s — model busy but not delivering (hard digest stuck at 2621efcc102c5c56)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 913816 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 0, "score": 0, "scoreMax": null, "signals": [], "failReason": null } } ] }, "quality": { "score": 0, "ruleId": "Q_ZERO_SCORE_UNKNOWN_MAX", "summary": "The gate denominator is unavailable and the observed gate score is zero.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 913816 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 0, "score": 0, "scoreMax": null, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6020093441228713 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 913816 }, { "source": "facts", "path": "outcome.durationMs", "value": 3382779 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3, "band": "framework-gap" }, "facts": { "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 571, "peakRssMb": 10336, "peakCpuPercent": 1236.3, "firstAtMs": 0, "lastAtMs": 3367548 }, "gpu": { "available": true, "sampleCount": 571, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 571, "peakUsedMb": 27180, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 7753, "totalOutputTokens": 5757, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T04:12:14.280Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3368998, "meanTokensPerSec": 2.83, "genTokensPerSec": 2.83, "genTokens": 5757, "promptTokensPerSec": 146.05, "peakRssMb": 10336, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T03:16:05.332Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T03:16:05.240Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.05633544921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.05633544921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1251s — model busy but not delivering (hard digest stuck at 2621efcc102c5c56)", "durationMs": 3382779, "timeoutMs": 5619147, "budgetUsedFraction": 0.6020093441228713 }, "timing": { "startedAt": "2026-11-04T03:15:59.498Z", "finishedAt": "2026-11-04T04:12:22.277Z", "timeToFirstArtifactMs": 913816, "timeToLastArtifactWriteMs": 2090520, "timeToFirstTokenMs": 44808, "firstTurnTtftMs": 34047, "timeToFirstToolCallMs": 90076 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "list_dir": 1, "read_file": 4, "write_file": 3, "run_installed_script": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 7358, "filePath": "data-wrangle", "bytes": 0, "score": 0, "scoreMax": null, "signals": [], "failReason": null } ], "latest": { "filePath": "data-wrangle", "bytes": 0, "score": 0, "scoreMax": null, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T03:15:59.502Z] [trial] id=data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw scenario=data-wrangle model=underdog-saluki-27b-iq2-mix", "[2026-11-04T03:15:59.502Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-data-wrangle-hjkOuL", "[2026-11-04T03:15:59.502Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "[2026-11-04T03:15:59.503Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T03:15:59.503Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T03:15:59.504Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T03:15:59.506Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T03:15:59.512Z] [trial] throughput-scaled ceiling: 45m → 94m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T03:15:59.512Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-data-wrangle-hjkOuL", "[2026-11-04T03:16:05.235Z] [trial] daemon spawned pid=34812 port=https://127.0.0.1:51509", "[2026-11-04T03:16:06.263Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T03:16:06.282Z] [trial] running scenario setup hook", "[2026-11-04T03:16:06.281Z] [trial] meester=rosalind", "[2026-11-04T03:16:06.767Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T03:16:06.766Z] [trial] scenario setup complete", "[2026-11-04T03:16:06.846Z] [poll] initial digests hard=cbe0e8fd58a2cb43 soft=b37c4aadb3c17e33", "[2026-11-04T03:16:06.902Z] [poll] hard-progress: cbe0e8fd58a2cb43 → 8c40c5c9059e2bc9 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T03:17:35.061Z] [poll] hard-progress: 8c40c5c9059e2bc9 → 93cc71451aac55c6 after 88s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=60,stream=87) sniff=0/0B)", "[2026-11-04T03:31:18.438Z] [poll] hard-progress: 93cc71451aac55c6 → 23aec57ecb90b62f after 823s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=2212,stream=3109) sniff=0/0B)", "[2026-11-04T03:37:36.255Z] [poll] hard-progress: 23aec57ecb90b62f → a91b5f2ea81e8c51 after 378s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=3171,stream=4981) sniff=0/0B)", "[2026-11-04T03:38:05.782Z] [poll] hard-progress: a91b5f2ea81e8c51 → 0c99b8b845c82439 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=3218,stream=5039) sniff=0/0B)", "[2026-11-04T03:50:53.503Z] [poll] hard-progress: 0c99b8b845c82439 → 618db438365f3cf8 after 768s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=5168,stream=7915) sniff=0/0B)", "[2026-11-04T03:51:22.969Z] [poll] hard-progress: 618db438365f3cf8 → 2621efcc102c5c56 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=5208,stream=7974) sniff=0/0B)", "[2026-11-04T04:12:13.791Z] [poll] no-progress (hard): no real progress for 1251s (threshold=1249s); model busy but not delivering — failing trial", "[2026-11-04T04:12:14.344Z] [capture] dumped 1 session(s)" ] }, "metrics": { "process": { "sampleCount": 571, "peakRssMb": 10336, "peakCpuPercent": 1236.3, "firstAtMs": 0, "lastAtMs": 3367548 }, "gpu": { "available": true, "sampleCount": 571, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 571, "peakUsedMb": 27180, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 7753, "totalOutputTokens": 5757, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T04:12:14.280Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3368998, "meanTokensPerSec": 2.83, "genTokensPerSec": 2.83, "genTokens": 5757, "promptTokensPerSec": 146.05, "peakRssMb": 10336, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T03:15:59.498Z", "finishedAt": "2026-11-04T04:12:22.277Z", "timeToFirstArtifactMs": 913816, "timeToLastArtifactWriteMs": 2090520, "timeToFirstTokenMs": 44808, "firstTurnTtftMs": 34047, "timeToFirstToolCallMs": 90076 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T03:16:05.332Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T03:16:05.240Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.05633544921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.05633544921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-data-wrangle", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "scenarioId": "data-wrangle", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/workspace/customer-data-cleanup/out/customers.json", "reason": "Required output was not committed before watchdog termination; no artifact score invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:56:49.216Z", "reviewId": "007", "cellId": "r1-saluki-cuda-core-data-wrangle", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "scenarioId": "data-wrangle", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "baselineCellId": "r1-saluki-cuda-core-data-wrangle", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "ab0c7b9efe390b1155b670c302fb2e50a0a169294812a614bb54a37dd5e9dea2", "evidenceSha256": "4a3fe0f7b2147cfa253063e6a689fa7f509410b52652ca90c83ffb584669982f", "artifactFiles": [], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw\\workspace\\customer-data-cleanup\\data\\raw\\customers_a.csv", "bytes": 1045, "sha256": "bc92617373571bdf0bef1c29ee05aad44a7a68274c0f7334b1ebe8aa5eedfe54", "shownBytes": 1045, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw\\workspace\\customer-data-cleanup\\data\\raw\\customers_b.csv", "bytes": 1087, "sha256": "ff167222077dd5a971ee88d6ab656d98b2b2051e84ceb223bcc893aab9186b09", "shownBytes": 1087, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw\\workspace\\customer-data-cleanup\\data\\raw\\legacy_export.csv", "bytes": 1148, "sha256": "90e6abcfcf404de96b3ac6385e3ddc3265125552377f3c3428d5636d310f9afa", "shownBytes": 1148, "truncated": false } ], "scoreAxes": {}, "justification": "The declared deliverable workspace/customer-data-cleanup/out/customers.json was missing at closeout, and the artifacts list is empty. Only the three raw input CSVs (BOM, quoted commas, legacy banner, near-duplicate emails) were captured. There is no transcript, script or validator output showing why the file was never written.", "limitations": "The evidence has no run logs or tool calls, so the cause is unknown. It could be an interruption, a failed script or a wrong output path. This does not establish a model failure or a success. No scores were assigned and nothing was run or retried.", "meanScore": null, "durationMs": 14940, "costUsd": 0.02297, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4974, "cache_read_input_tokens": 0, "output_tokens": 307, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4974, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 307, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 4974, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4974 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 307, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 4974, "webSearchRequests": 0, "costUSD": 0.02297, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4974, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - data-wrangle / underdog-saluki-27b-iq2-mix\n\n**Composite: 3.0 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 56m 23s; budget used: 60.2%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1251s — model busy but not delivering (hard digest stuck at 2621efcc102c5c56)\"`; `facts.timing.timeToFirstArtifactMs: 913816`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":0,\"score\":0,\"scoreMax\":null,\"signals\":[],\"failReason\":null}` |\n| Output quality (25%) | 0 / 10 | The gate denominator is unavailable and the observed gate score is zero. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 913816`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":0,\"score\":0,\"scoreMax\":null,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.6020093441228713`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 913816`; `facts.outcome.durationMs: 3382779` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 0 + 0.2 * 7.5 + 0.15 * 10 = **3.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10336 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27180 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.83 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 7753 + 5757 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1251s — model busy but not delivering (hard digest stuck at 2621efcc102c5c56)\"`\n- `facts.timing.timeToFirstArtifactMs: 913816`\n- `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":0,\"score\":0,\"scoreMax\":null,\"signals\":[],\"failReason\":null}`\n\n### Output quality - Q_ZERO_SCORE_UNKNOWN_MAX\n\nThe gate denominator is unavailable and the observed gate score is zero.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 913816`\n- `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":0,\"score\":0,\"scoreMax\":null,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.6020093441228713`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 913816`\n- `facts.outcome.durationMs: 3382779`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1251s — model busy but not delivering (hard digest stuck at 2621efcc102c5c56)\"`; `facts.timing.timeToFirstArtifactMs: 913816`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":0,\"score\":0,\"scoreMax\":null,\"signals\":[],\"failReason\":null}`\n- Quality: The gate denominator is unavailable and the observed gate score is zero. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 913816`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":0,\"score\":0,\"scoreMax\":null,\"signals\":[],\"failReason\":null}`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.6020093441228713`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 913816`; `facts.outcome.durationMs: 3382779`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory review and outcome\n\nThe required out/customers.json is absent. Record noArtifact with no qualitative scores. A 3142-byte helper, tools/clean_customers.mjs, was written three times, but no completed execution produced the requested dataset. The existing raw CSVs and check helper are fixtures, not delivered output. Read the complete cleaning helper; its unexecuted contents do not establish correctness of the eleven output properties. Fixed composite remains 3.0.\n\n## Positives\n\n- The model read the three source exports and made real workspace writes (toolUse.byTool.read_file: 4; write_file: 3; timing.timeToFirstArtifactMs: 913816). This was not a failure to load or invoke filesystem tools.\n- The expected Developer role was present and no fixed-rubric red flag or user intervention was recorded (team.missingExpectedRoles: []; toolUse.redFlags: []; autoAnswer.total: 0). These clean behavioral fields do not imply task completion.\n\n## Negatives\n\n- The required dataset never appeared: sniff.latest.bytes and score are both zero. The hard-progress watchdog ended the trial after 1251 seconds without a completed tool/action change, at 56m23s total (outcome.failureMode: model-stuck; outcome.durationMs: 3382779).\n- The one run_installed_script call tried name node and failed because no such installed project script exists. Its error explicitly distinguished installed scripts from local files and offered direct write_file output when execution is unavailable (toolUse.byTool.run_installed_script: 1; recording/chat-events.jsonl at 03:51:17Z). The model-facing roster lacked run_nodejs_script. This is tool-surface/recovery evidence, not evidence that system Node needs installation.\n- The final unfinished request continued reasoning until the watchdog stopped it; the last completed write was at 03:50:49Z (timing.timeToLastArtifactWriteMs: 2090520; session-telemetry.json inflight: true; advisory-generation-evidence.json). No capability retry or manual helper execution was performed.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/de/developer/versions/1.5.1/about.md, add to “Finish the whole feature path”: “When the requested deliverable is data, complete the data file; a helper is only an intermediate step. Check that the current tools can execute a helper before spending the turn refining it.” This describes a workflow invariant without listing particular tools. Measure it on two unrelated data-production tasks, retaining the current model sampling and gates.\n\n## Strategic experiment — not applied\n\nIn packages/mcp/src/installed-script-miss.ts, base canRunFiles/canDerive on the effective per-turn allowlist, not only the session roster; the current message suggested a runner that the current 12-tool surface did not expose. Also investigate forwarding the written helper path from input.args into script-file diagnostics. Keep execution behind existing workspace authority and permissions. Test the routing improvement with a second file-derived output task; do not grant new execution capabilities merely to pass this case.\n\n## Generalization and historical interpretation\n\nThe failure pattern is creating implementation machinery without producing the requested result, followed by an unproductive recovery turn. Petshop independently showed asset-production/tool-selection trouble, while schema-migration and symptom-debug delivered direct edits successfully. Compare these task shapes before attributing the outcome to GPU capacity or parameter count. Older data-wrangle trials below also failed, but produced datasets with row-count/dedup errors; this run failed earlier in the delivery chain. A concrete next experiment is the generic deliverable-completion wording above across two data tasks, measuring produced-output rate and time, without changing the current campaign.\n\n## Memory and execution\n\nPeak VRAM 10626/12227 MiB; process RSS 10336 MiB; whole-system memory 27180/31865 MiB (perf.gpu, perf.process, perf.systemMemory). Logs retain CUDA/native 0.1.57, 65536 context tokens, one slot, f16 KV, dense FFN of 39/64 layers on GPU and 25 on CPU. No explicit OOM, context overflow or unexpected native exit was found. Shutdown SIGTERM was expected after the recorded verdict. Shared GPU-memory use is not separately measured. Installed-app payload/admission warnings refer to a different installed app build; the trial explicitly launched the verified workspace CUDA binary and used local admission.\n\n## Last three same-scenario trials\n\n- [data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/facts.json): fail, fixed 3.4, 555.3 seconds, recorded decode 141.01 tokens/sec.\n- [data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k/facts.json): fail, fixed 3.4, 726.4 seconds, recorded decode 135.74 tokens/sec.\n- [data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-32-18-500Z-r0tk/facts.json): fail, fixed 5.6, 265.8 seconds, recorded decode 147.47 tokens/sec.\n\nHistorical context only: these trials used Gemma4 12B/native 0.1.54, not the current frozen identity.\n" }, { "cellId": "r1-saluki-cuda-core-incident-postmortem", "baselineCellId": "r1-saluki-cuda-core-incident-postmortem", "suite": "core", "scenario": "incident-postmortem", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T04:15:17.174Z", "finishedAt": "2026-11-04T04:49:20.722Z", "durationMs": 2043548, "success": false, "reason": "no real progress for 1875s — model busy but not delivering (hard digest stuck at f23b2dac5bc6a586)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "incident-postmortem", "score": 0, "bytes": 0 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1875s — model busy but not delivering (hard digest stuck at f23b2dac5bc6a586)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "quality": { "score": 0, "ruleId": "Q_NO_PRODUCED_OUTPUT", "summary": "No authored artifact event or scored sniff output was observed.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.4091353462412987 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 1 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 2043548 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.5, "band": "framework-gap" }, "facts": { "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 345, "peakRssMb": 9472, "peakCpuPercent": 1206.5, "firstAtMs": 1, "lastAtMs": 2029483 }, "gpu": { "available": true, "sampleCount": 345, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 345, "peakUsedMb": 27219, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11311, "totalOutputTokens": 195, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T04:49:14.739Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2030912, "meanTokensPerSec": 2.76, "genTokensPerSec": 2.76, "genTokens": 195, "promptTokensPerSec": 166.69, "peakRssMb": 9472, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T04:15:23.424Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T04:15:23.293Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "jordan": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1457061767578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1457061767578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1875s — model busy but not delivering (hard digest stuck at f23b2dac5bc6a586)", "durationMs": 2043548, "timeoutMs": 4994797, "budgetUsedFraction": 0.4091353462412987 }, "timing": { "startedAt": "2026-11-04T04:15:17.174Z", "finishedAt": "2026-11-04T04:49:20.722Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 90677, "firstTurnTtftMs": 78296, "timeToFirstToolCallMs": 161382 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 1, "byTool": { "read_files": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [], "latest": null }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T04:15:17.179Z] [trial] id=incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek scenario=incident-postmortem model=underdog-saluki-27b-iq2-mix", "[2026-11-04T04:15:17.179Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "[2026-11-04T04:15:17.179Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-incident-postmortem-EgRdNS", "[2026-11-04T04:15:17.179Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T04:15:17.179Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T04:15:17.182Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T04:15:17.186Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T04:15:17.192Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-incident-postmortem-EgRdNS", "[2026-11-04T04:15:17.192Z] [trial] throughput-scaled ceiling: 40m → 83m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T04:15:23.358Z] [trial] daemon spawned pid=27656 port=https://127.0.0.1:53185", "[2026-11-04T04:15:24.736Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T04:15:24.754Z] [trial] meester=zara", "[2026-11-04T04:15:24.754Z] [trial] running scenario setup hook", "[2026-11-04T04:15:25.297Z] [trial] scenario setup complete", "[2026-11-04T04:15:25.299Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T04:15:25.374Z] [poll] initial digests hard=e9b411fcf4190b29 soft=98ef7c28f4505478", "[2026-11-04T04:15:25.472Z] [poll] hard-progress: e9b411fcf4190b29 → b13769ef70321c60 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T04:17:59.304Z] [poll] hard-progress: b13769ef70321c60 → f23b2dac5bc6a586 after 154s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=184,stream=231) sniff=0/0B)", "[2026-11-04T04:49:14.266Z] [poll] no-progress (hard): no real progress for 1875s (threshold=1873s); model busy but not delivering — failing trial", "[2026-11-04T04:49:14.783Z] [capture] dumped 1 session(s)", "[2026-11-04T04:49:14.786Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T04:49:14.787Z] [capture] INCOMPLETE TRANSCRIPT for 1 session(s) — jordan: 0 persisted vs 1 recorded. Assistant turns commit at turn END, so a mid-turn termination loses them. Triage from session-telemetry.json and daemon.log, NOT from the empty transcript.", "[2026-11-04T04:49:14.851Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T04:49:20.720Z] [trial] daemon shut down", "[2026-11-04T04:49:20.734Z] [trial] FAIL duration=2043548ms reason=no real progress for 1875s — model busy but not delivering (hard digest stuck at f23b2dac5bc6a586)" ] }, "metrics": { "process": { "sampleCount": 345, "peakRssMb": 9472, "peakCpuPercent": 1206.5, "firstAtMs": 1, "lastAtMs": 2029483 }, "gpu": { "available": true, "sampleCount": 345, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 345, "peakUsedMb": 27219, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11311, "totalOutputTokens": 195, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T04:49:14.739Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2030912, "meanTokensPerSec": 2.76, "genTokensPerSec": 2.76, "genTokens": 195, "promptTokensPerSec": 166.69, "peakRssMb": 9472, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T04:15:17.174Z", "finishedAt": "2026-11-04T04:49:20.722Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 90677, "firstTurnTtftMs": 78296, "timeToFirstToolCallMs": 161382 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T04:15:23.424Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T04:15:23.293Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "jordan": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1457061767578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1457061767578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-incident-postmortem", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "scenarioId": "incident-postmortem", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/workspace/checkout-incident-postmortem/postmortem.md", "reason": "Required output was not committed before watchdog termination; no artifact score invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:00.275Z", "reviewId": "008", "cellId": "r1-saluki-cuda-core-incident-postmortem", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "scenarioId": "incident-postmortem", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "baselineCellId": "r1-saluki-cuda-core-incident-postmortem", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "ccf84123c95f7cf87046211d16812bc609b0c4e3b0b63426b45aa73d18d6392f", "evidenceSha256": "6cfed4d266ec6a84655ffca054f3d542e53ec56f63e5cfb31a753691e9625357", "artifactFiles": [], "contextFiles": [], "scoreAxes": {}, "justification": "No postmortem.md existed at closeout: the declared path workspace/checkout-incident-postmortem/postmortem.md is missing, artifacts is empty, and no context files were captured. The disposition is runner-terminal, so the run ended before any deliverable was written. The evidence has no transcript or tool trace showing why.", "limitations": "The evidence has no tool-call log, transcript or error message. I can't tell whether the evidence reads finished, whether write_file was attempted, or what ended the run. I can't judge postmortem quality, and I assign no scores and no model fault or success.", "meanScore": null, "durationMs": 15102, "costUsd": 0.010438, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1821, "cache_read_input_tokens": 700, "output_tokens": 308, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1821, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 308, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1821, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1821 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 308, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1821, "webSearchRequests": 0, "costUSD": 0.010438, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1821, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - incident-postmortem / underdog-saluki-27b-iq2-mix\n\n**Composite: 3.5 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 34m 4s; budget used: 40.9%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1875s — model busy but not delivering (hard digest stuck at f23b2dac5bc6a586)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null` |\n| Output quality (25%) | 0 / 10 | No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.4091353462412987`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 2043548` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 0 + 0.2 * 10 + 0.15 * 10 = **3.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9472 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27219 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.76 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 11311 + 195 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1875s — model busy but not delivering (hard digest stuck at f23b2dac5bc6a586)\"`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: null`\n\n### Output quality - Q_NO_PRODUCED_OUTPUT\n\nNo authored artifact event or scored sniff output was observed.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: null`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.4091353462412987`\n- `facts.toolUse.totalToolCalls: 1`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 2043548`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1875s — model busy but not delivering (hard digest stuck at f23b2dac5bc6a586)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null`\n- Quality: No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.4091353462412987`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 2043548`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory review and outcome\n\nThe required workspace postmortem.md does not exist. Record noArtifact; do not judge the harness-generated postmortem.md at the trial root as though it were model output. The five evidence files are fixtures. Fixed composite remains 3.5, with completion and quality both zero. No qualitative score can be assigned to an unfinished tool argument.\n\n## Positives\n\n- A successful read_files call fetched all five evidence files in one batch (toolUse.byTool.read_files: 1; recording/chat-events.jsonl at 04:17:58Z). The Researcher role and source access were available (team.missingExpectedRoles: []).\n- The engine continued producing tokens and eventually began a write_file argument stream; this was neither a failed load nor a dead engine (session-telemetry.json inflight: true; advisory-generation-evidence.json; daemon.log).\n\n## Negatives\n\n- No complete artifact was committed before the 1875-second hard-progress watchdog fired (timing.timeToFirstArtifactMs: null; outcome.failureMode: model-stuck; outcome.durationMs: 2043548). The total was 34m4s, despite the larger absolute timeout.\n- The read-to-write interval consumed most of the delivery window: reasoning deltas continued until 04:44:25Z; write_file argument streaming began 04:44:29Z and was still incomplete at 04:49:14Z, after only 2155 streamed argument characters (advisory-generation-evidence.json). No file was written. This is a failure to deliver within the unchanged hosting/eval budget, not proof the model could never write a grounded report.\n- The fixed performance summary includes only 195 generated tokens at 2.76 tokens/sec from completed timing records (perf.derived). The unfinished request reached at least 4849 output tokens at approximately 2.63 tokens/sec in live engine telemetry. The 195 value must not be presented as the entire trial's generation. The captured persisted transcript is also explicitly incomplete because termination happened mid-turn (log.txt; session-telemetry.json).\n\n## Tactical experiment — not applied\n\nFor a later isolated A/B, reduce tuning.reasoning.thinkingBudget and tuning.profiles.thinking-general.reasoning.thinkingBudget from 4096 to 1024 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json. Keep maxTokens, temperature, context and deadlines fixed. At roughly 2.63 tokens/sec, a 4096-token reasoning allowance alone can occupy about 26 minutes. Test whether the smaller allowance leaves time to finish the first write, using this report task and an unrelated plan plus coding regression. Reject if grounding or correctness degrades. No change is made to this campaign.\n\n## Strategic experiment — not applied\n\nIn packages/core/src/local-loop/constrained-turn.ts, explore a first-deliverable reasoning allowance that accounts for measured throughput and remaining write time, while preserving required output length and normal workspace writes. In evals/src/progress-fingerprint.ts/runner.ts add diagnostic classification for reasoning versus in-progress tool-argument assembly at watchdog termination. Keep hard/soft progress semantics and gates unchanged for the current comparison: partial text is still not delivered output. The existing atomic tool call must complete before it can produce a file; reducing pre-write reasoning is a safer hypothesis than counting endless thought as success.\n\n## Generalization and strategic note\n\nThis run and data-wrangle both remained alive without a required deliverable, while ops-runbook-anomaly completed a sequence of small writes at the same approximately 2.8 tokens/sec. This supports investigating long first-write latency and constrained-turn budgets across task types. The older three report trials passed, but a different model/native release and much higher throughput prevent a controlled regression claim. Next experiment: the 1024 reasoning cap with unchanged document gates, evaluated on report, planning and coding work in a separate campaign.\n\n## Memory and execution\n\nPeak VRAM 10630/12227 MiB; process RSS 9472 MiB; whole-system memory 27219/31865 MiB (perf.gpu, perf.process, perf.systemMemory). Logs retain CUDA/native 0.1.57, 65536 context tokens, one slot, f16 KV, dense FFN of 39/64 layers on GPU and 25 on CPU. No explicit OOM, context overflow or unexpected native exit was found. Shutdown SIGTERM was expected after the recorded verdict. Shared GPU-memory use is not separately measured. Installed-app payload/admission warnings refer to a different installed app build; the trial explicitly launched the verified workspace CUDA binary and used local admission.\n\n## Last three same-scenario trials\n\n- [incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/facts.json): pass, fixed 10, 119.7 seconds, recorded decode 101.82 tokens/sec.\n- [incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6/facts.json): pass, fixed 10, 113.5 seconds, recorded decode 106.14 tokens/sec.\n- [incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-58-11-732Z-41g4/facts.json): pass, fixed 10, 97.5 seconds, recorded decode 107.52 tokens/sec.\n\nHistorical context only: these trials used Gemma4 12B/native 0.1.54, not the current frozen identity.\n" }, { "cellId": "r1-saluki-cuda-core-ops-runbook-anomaly", "baselineCellId": "r1-saluki-cuda-core-ops-runbook-anomaly", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T04:51:33.588Z", "finishedAt": "2026-11-04T05:03:26.738Z", "durationMs": 713150, "success": true, "reason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 170174 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "ops-runbook-anomaly", "bytes": 704, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.1631755287273819 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 13 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 170174 }, { "source": "facts", "path": "outcome.durationMs", "value": 713150 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 119, "peakRssMb": 13064, "peakCpuPercent": 1312, "firstAtMs": 1, "lastAtMs": 700107 }, "gpu": { "available": true, "sampleCount": 119, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 119, "peakUsedMb": 30355, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 22760, "totalOutputTokens": 1424, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18159, "todayTokensCached": 12407, "todayTokensOut": 429, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18159, "totalTokensCached": 12407, "totalTokensOut": 429, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5515, "outputTokens": 95, "cost": 0, "durationMs": 117072, "outputTokensPerSec": 2.8867916108975895, "at": "2026-11-04T05:03:16.069Z" }, "lastUpdated": "2026-11-04T05:03:16.069Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T05:03:20.978Z" } }, "source": "engine-log" }, "derived": { "durationMs": 701560, "meanTokensPerSec": 2.8, "genTokensPerSec": 2.8, "genTokens": 1424, "promptTokensPerSec": 149.79, "peakRssMb": 13064, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 18159, "output": 429 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T04:51:39.419Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T04:51:39.320Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "onno": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 5, "maxContextFill": 0.0971221923828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0971221923828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "durationMs": 713150, "timeoutMs": 4370447, "budgetUsedFraction": 0.1631755287273819 }, "timing": { "startedAt": "2026-11-04T04:51:33.588Z", "finishedAt": "2026-11-04T05:03:26.738Z", "timeToFirstArtifactMs": 170174, "timeToLastArtifactWriteMs": 702481, "timeToFirstTokenMs": 86165, "firstTurnTtftMs": 75287, "timeToFirstToolCallMs": 117706 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 13, "byTool": { "read_file": 8, "write_file": 3, "append_to_file": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 10 }, "sniff": { "progression": [ { "atMs": 174871, "filePath": "ops-runbook-anomaly", "bytes": 32, "score": 2, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded" ], "failReason": "runlog.md is missing step 2 (manifest checksum)'s observed verification value under its own \"STEP 2\" heading — read state/manifest-checksum.txt and checks/expected-counts.md, then verify they match, then record the value verbatim in execution order" }, { "atMs": 240262, "filePath": "ops-runbook-anomaly", "bytes": 57, "score": 3, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded" ], "failReason": "runlog.md is missing step 3 (config version)'s observed verification value under its own \"STEP 3\" heading — read the version field from state/config.json, then record the value verbatim in execution order" }, { "atMs": 305033, "filePath": "ops-runbook-anomaly", "bytes": 85, "score": 4, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded" ], "failReason": "runlog.md must record STEP 1, STEP 2, STEP 3, then STEP 4 in execution order" }, { "atMs": 458010, "filePath": "ops-runbook-anomaly", "bytes": 685, "score": 5, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "halt-report-grounded" ], "failReason": "runlog.md must record STEP 1, STEP 2, STEP 3, then STEP 4 in execution order" }, { "atMs": 706863, "filePath": "ops-runbook-anomaly", "bytes": 704, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } ], "latest": { "filePath": "ops-runbook-anomaly", "bytes": 704, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T04:51:33.592Z] [trial] id=ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9 scenario=ops-runbook-anomaly model=underdog-saluki-27b-iq2-mix", "[2026-11-04T04:51:33.592Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-ops-runbook-anomaly-uCaYGh", "[2026-11-04T04:51:33.592Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "[2026-11-04T04:51:33.592Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T04:51:33.592Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T04:51:33.595Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T04:51:33.600Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T04:51:33.606Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-ops-runbook-anomaly-uCaYGh", "[2026-11-04T04:51:33.606Z] [trial] throughput-scaled ceiling: 35m → 73m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T04:51:39.345Z] [trial] daemon spawned pid=7512 port=https://127.0.0.1:54134", "[2026-11-04T04:51:40.284Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T04:51:40.301Z] [trial] meester=linnea", "[2026-11-04T04:51:40.302Z] [trial] running scenario setup hook", "[2026-11-04T04:51:40.783Z] [trial] scenario setup complete", "[2026-11-04T04:51:40.784Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T04:51:40.866Z] [poll] initial digests hard=0d8eeec20c02bbc7 soft=e75f6b9e7e130222", "[2026-11-04T04:51:40.947Z] [poll] hard-progress: 0d8eeec20c02bbc7 → b52e0b0d3a155d7e after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T04:53:34.749Z] [poll] hard-progress: b52e0b0d3a155d7e → 24b86de6614da721 after 114s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=80,stream=77) sniff=0/0B)", "[2026-11-04T04:53:58.686Z] [poll] hard-progress: 24b86de6614da721 → 55c603319120ba54 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=109,stream=114) sniff=0/0B)", "[2026-11-04T04:54:28.534Z] [poll] hard-progress: 55c603319120ba54 → 577054b5560b6021 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=151,stream=177) sniff=2/32B)", "[2026-11-04T04:55:04.059Z] [poll] hard-progress: 577054b5560b6021 → beffb5b78642b798 after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=204,stream=249) sniff=2/32B)", "[2026-11-04T04:55:34.009Z] [poll] hard-progress: beffb5b78642b798 → fe218f4f5118e564 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=242,stream=314) sniff=3/57B)", "[2026-11-04T04:56:03.468Z] [poll] hard-progress: fe218f4f5118e564 → 043edd45f75b3f76 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=291,stream=370) sniff=3/57B)", "[2026-11-04T04:56:38.725Z] [poll] hard-progress: 043edd45f75b3f76 → c6de6fe9f59f7ede after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=346,stream=451) sniff=4/85B)", "[2026-11-04T04:57:02.073Z] [poll] hard-progress: c6de6fe9f59f7ede → fd5cfd4b38ad04a6 after 23s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=385,stream=497) sniff=4/85B)" ] }, "metrics": { "process": { "sampleCount": 119, "peakRssMb": 13064, "peakCpuPercent": 1312, "firstAtMs": 1, "lastAtMs": 700107 }, "gpu": { "available": true, "sampleCount": 119, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 119, "peakUsedMb": 30355, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 22760, "totalOutputTokens": 1424, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18159, "todayTokensCached": 12407, "todayTokensOut": 429, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18159, "totalTokensCached": 12407, "totalTokensOut": 429, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5515, "outputTokens": 95, "cost": 0, "durationMs": 117072, "outputTokensPerSec": 2.8867916108975895, "at": "2026-11-04T05:03:16.069Z" }, "lastUpdated": "2026-11-04T05:03:16.069Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T05:03:20.978Z" } }, "source": "engine-log" }, "derived": { "durationMs": 701560, "meanTokensPerSec": 2.8, "genTokensPerSec": 2.8, "genTokens": 1424, "promptTokensPerSec": 149.79, "peakRssMb": 13064, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 18159, "output": 429 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T04:51:33.588Z", "finishedAt": "2026-11-04T05:03:26.738Z", "timeToFirstArtifactMs": 170174, "timeToLastArtifactWriteMs": 702481, "timeToFirstTokenMs": 86165, "firstTurnTtftMs": 75287, "timeToFirstToolCallMs": 117706 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T04:51:39.419Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T04:51:39.320Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "onno": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 5, "maxContextFill": 0.0971221923828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0971221923828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "scenarioId": "ops-runbook-anomaly", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/workspace/quarterly-maintenance-window/halt-report.md", "reason": "No declared judge axes or legacy HTML artifact; full procedure outputs reviewed in postmortem without invented scores." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:02.625Z", "reviewId": "009", "cellId": "r1-saluki-cuda-core-ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "scenarioId": "ops-runbook-anomaly", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "baselineCellId": "r1-saluki-cuda-core-ops-runbook-anomaly", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "6076a420223cabb4a792f13ff52743eea505fb9b7cf412db4cd640259292fdf7", "evidenceSha256": "0b97785d282c4c9e846b90c5fc556102c6433586fcf592da4d495578a295bda5", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/workspace/quarterly-maintenance-window/halt-report.md", "bytes": 606, "sha256": "e04f62b6bca209d99b805ac5da9ccd418dcbe2833ac55a52867bec315a7f28b4", "shownBytes": 606, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The halt-report.md is well-formed: it names Step 4, the expected value (fresh, under 24h), and the observed values (stale, 71h), including both status and age fields. It says Steps 5–6 were not executed. This is consistent, but not independently checked.", "limitations": "Only halt-report.md was shown. runlog.md, runbook.md and state files were not provided. I could not verify Steps 1–3 values, the read order, or that Steps 5–6 outputs were absent. The report's claims are taken as stated.", "meanScore": null, "durationMs": 14986, "costUsd": 0.010454000000000001, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1855, "cache_read_input_tokens": 700, "output_tokens": 296, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1855, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 296, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1855, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1855 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 296, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1855, "webSearchRequests": 0, "costUSD": 0.010454000000000001, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1855, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - ops-runbook-anomaly / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 11m 53s; budget used: 16.3%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 170174`; `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":704,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",\"...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.1631755287273819`; `facts.toolUse.totalToolCalls: 13`; `facts.timing.timeToFirstArtifactMs: 170174`; `facts.outcome.durationMs: 713150` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13064 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30355 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.8 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 22760 + 1424 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 10% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 170174`\n- `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":704,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",\"...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.1631755287273819`\n- `facts.toolUse.totalToolCalls: 13`\n- `facts.timing.timeToFirstArtifactMs: 170174`\n- `facts.outcome.durationMs: 713150`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 170174`; `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":704,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",\"...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.1631755287273819`; `facts.toolUse.totalToolCalls: 13`; `facts.timing.timeToFirstArtifactMs: 170174`; `facts.outcome.durationMs: 713150`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nThe scenario declares no judge axes and does not use the legacy HTML rubric. Record notApplicable without invented scores. Read the full runbook, 104-byte runlog and 606-byte halt report, plus backup state. The record contains STEP 1 through STEP 4 in order, preserves the exact service count/checksum/config values, and reports stale backup status with age 71 hours against fresh/under 24. The report accurately says later maintenance and decommission steps were not executed; the deterministic gate verifies sentinel absence and ordered source reads. Fixed composite remains 9.5.\n\n## Positives\n\n- All seven behavioral signals passed, including stop-on-anomaly, ordered records, grounded reporting, source reads and no phantom completion (sniff.latest.score: 7/7; outcome.success: true). The model halted the procedure appropriately.\n- The deliverable arrived incrementally, with first output after 170174 ms and no user intervention or red flags (timing.timeToFirstArtifactMs; autoAnswer.total: 0; toolUse.redFlags: []). This contrasts with the two long no-deliverable stalls.\n\n## Negatives\n\n- The first log omitted explicit STEP headings and the failed step's entry, requiring repair after the report was already written (toolUse.byTool.write_file: 3; append_to_file: 2; sniff progression in log.txt from 4/7 to 5/7 to 7/7). The final result is correct, but not first-pass compliant.\n- Thirteen calls and 713150 ms were required, at 2.80 decode and 149.79 prompt tokens/sec (toolUse.totalToolCalls; outcome.durationMs; perf.derived). The fixed efficiency score measures count and scaled budget, not responsiveness.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/re/researcher/versions/1.2.0/about.md, add: “For a procedure, record each requested step label and observed verification value immediately, including the step that causes a halt.” This is a generic recordkeeping instruction, not a copy of the fixture's headings or expected values. Test on another ordered checklist and an unrelated evidence-summary task, measuring first-pass compliance and repair count.\n\n## Strategic experiment — not applied\n\nIn a proposed packages/service/src/tasks/procedure-record.ts, provide a reusable structured procedure record that stores step identity, source, observation and halt reason together, then renders the requested log. Integrate only through existing task/store write boundaries and preserve user-specific formats. This would reduce independent narrative/log drift; the observed correction of missing step labels motivates a prototype, not a proven improvement. Compare a second runbook and a non-operational checklist before adoption.\n\n## Generalization and next experiment\n\nThe same Researcher role that failed to finish the long incident report successfully handled short sequential records and halted on evidence. Successful loading and role provisioning therefore do not alone predict delivery on long synthesis tasks. A separate A/B of generic procedural recordkeeping should preserve this halt behavior while reducing the three write_file calls; validate on an unrelated checklist. Historical trials below reached the same 9.5 fixed score at much lower elapsed time, but different model/engine conditions prohibit a causal speed comparison.\n\n## Memory and execution\n\nPeak VRAM 10638/12227 MiB; process RSS 13064 MiB; whole-system memory 30355/31865 MiB (perf.gpu, perf.process, perf.systemMemory). Logs retain CUDA/native 0.1.57, 65536 context tokens, one slot, f16 KV, dense FFN of 39/64 layers on GPU and 25 on CPU. No explicit OOM, context overflow or unexpected native exit was found. Shutdown SIGTERM was expected after the recorded verdict. Shared GPU-memory use is not separately measured. Installed-app payload/admission warnings refer to a different installed app build; the trial explicitly launched the verified workspace CUDA binary and used local admission.\n\n## Last three same-scenario trials\n\n- [ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/facts.json): pass, fixed 9.5, 74.8 seconds, recorded decode 115.02 tokens/sec.\n- [ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje/facts.json): pass, fixed 9.5, 85.8 seconds, recorded decode 122.09 tokens/sec.\n- [ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-03-46-986Z-mfu0/facts.json): pass, fixed 9.5, 74.3 seconds, recorded decode 122.07 tokens/sec.\n\nHistorical context only: these trials used Gemma4 12B/native 0.1.54, not the current frozen identity.\n" }, { "cellId": "r1-saluki-cuda-core-plan-and-estimate", "baselineCellId": "r1-saluki-cuda-core-plan-and-estimate", "suite": "core", "scenario": "plan-and-estimate", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T05:06:14.546Z", "finishedAt": "2026-11-04T05:14:49.275Z", "durationMs": 514729, "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 504298 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 3545, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.1648848657867323 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 504298 }, { "source": "facts", "path": "outcome.durationMs", "value": 514729 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 86, "peakRssMb": 9335, "peakCpuPercent": 1214.5, "firstAtMs": 0, "lastAtMs": 502147 }, "gpu": { "available": true, "sampleCount": 86, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 86, "peakUsedMb": 27375, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4521, "totalOutputTokens": 1306, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T05:14:44.733Z" } }, "source": "engine-log" }, "derived": { "durationMs": 503694, "meanTokensPerSec": 2.88, "genTokensPerSec": 2.88, "genTokens": 1306, "promptTokensPerSec": 162.7, "peakRssMb": 9335, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T05:06:20.663Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T05:06:20.561Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0569305419921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0569305419921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 514729, "timeoutMs": 3121748, "budgetUsedFraction": 0.1648848657867323 }, "timing": { "startedAt": "2026-11-04T05:06:14.546Z", "finishedAt": "2026-11-04T05:14:49.275Z", "timeToFirstArtifactMs": 504298, "timeToLastArtifactWriteMs": 504298, "timeToFirstTokenMs": 47052, "firstTurnTtftMs": 34729, "timeToFirstToolCallMs": 66244 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "read_files": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 509680, "filePath": "plan-and-estimate", "bytes": 3545, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } ], "latest": { "filePath": "plan-and-estimate", "bytes": 3545, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T05:06:14.550Z] [trial] id=plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9 scenario=plan-and-estimate model=underdog-saluki-27b-iq2-mix", "[2026-11-04T05:06:14.550Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "[2026-11-04T05:06:14.550Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-AsEmU0", "[2026-11-04T05:06:14.550Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T05:06:14.550Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T05:06:14.553Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T05:06:14.558Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T05:06:14.565Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-AsEmU0", "[2026-11-04T05:06:14.565Z] [trial] throughput-scaled ceiling: 25m → 52m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T05:06:20.556Z] [trial] daemon spawned pid=43576 port=https://127.0.0.1:64042", "[2026-11-04T05:06:21.965Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T05:06:21.985Z] [trial] meester=soren", "[2026-11-04T05:06:21.985Z] [trial] running scenario setup hook", "[2026-11-04T05:06:22.498Z] [trial] scenario setup complete", "[2026-11-04T05:06:22.500Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T05:06:22.584Z] [poll] initial digests hard=b11b76569ee3d205 soft=8e9f682aa556c460", "[2026-11-04T05:06:22.657Z] [poll] hard-progress: b11b76569ee3d205 → 43bc887ce31ba00b after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T05:07:21.609Z] [poll] hard-progress: 43bc887ce31ba00b → f82a167fb89c4349 after 59s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=42,stream=53) sniff=0/0B)", "[2026-11-04T05:14:44.226Z] [poll] terminal: success=true reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "[2026-11-04T05:14:44.778Z] [capture] dumped 1 session(s)", "[2026-11-04T05:14:44.781Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T05:14:44.840Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T05:14:49.273Z] [trial] daemon shut down", "[2026-11-04T05:14:49.284Z] [trial] PASS duration=514729ms reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)" ] }, "metrics": { "process": { "sampleCount": 86, "peakRssMb": 9335, "peakCpuPercent": 1214.5, "firstAtMs": 0, "lastAtMs": 502147 }, "gpu": { "available": true, "sampleCount": 86, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 86, "peakUsedMb": 27375, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4521, "totalOutputTokens": 1306, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T05:14:44.733Z" } }, "source": "engine-log" }, "derived": { "durationMs": 503694, "meanTokensPerSec": 2.88, "genTokensPerSec": 2.88, "genTokens": 1306, "promptTokensPerSec": 162.7, "peakRssMb": 9335, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T05:06:14.546Z", "finishedAt": "2026-11-04T05:14:49.275Z", "timeToFirstArtifactMs": 504298, "timeToLastArtifactWriteMs": 504298, "timeToFirstTokenMs": 47052, "firstTurnTtftMs": 34729, "timeToFirstToolCallMs": 66244 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T05:06:20.663Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T05:06:20.561Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0569305419921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0569305419921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "scenarioId": "plan-and-estimate", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/workspace/harbourview-office-relocation/plan.md", "artifactBytes": 3551, "reason": "Scenario has no declared judge axes or legacy HTML output. Full plan and source brief reviewed in postmortem; no qualitative scores invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:03.624Z", "reviewId": "010", "cellId": "r1-saluki-cuda-core-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "scenarioId": "plan-and-estimate", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9", "baselineCellId": "r1-saluki-cuda-core-plan-and-estimate", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b86ee59319b8edea5d182aa75fb8e62fcf03a473e25d980f27909cac337978cd", "evidenceSha256": "c25b0a4374b938864349fe651ac8b8f78b8e175ac1c94c149a627e43f44b6248", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/workspace/harbourview-office-relocation/plan.md", "bytes": 3551, "sha256": "b337dd9a43307ce859b08987892744be93ad8ff0bfd992e399412b68b8e19988", "shownBytes": 3551, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9\\workspace\\harbourview-office-relocation\\brief.md", "bytes": 732, "sha256": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "shownBytes": 732, "truncated": false } ], "scoreAxes": {}, "justification": "plan.md has the four sections in order and a table with the exact columns and 9 rows. Owners are from the roster, and Ola owns nothing. T3 depends on the T2 sign-off, and floor plan, movers and network are all covered. It is well under 1200 words. T4 depends on T1, not T2.", "limitations": "team.md was not shown, so the roster could not be checked directly. The final reply format (path and precis) is not in the evidence. Some details are unverified: the year 2026 is assumed, T8 has no link to T4 or T9, and T9 does not depend on T8. Source-only review.", "meanScore": null, "durationMs": 14393, "costUsd": 0.016862000000000002, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3412, "cache_read_input_tokens": 700, "output_tokens": 314, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3412, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 314, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3412, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3412 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 314, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3412, "webSearchRequests": 0, "costUSD": 0.016862000000000002, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3412, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - plan-and-estimate / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 8m 35s; budget used: 16.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 504298`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3545,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.1648848657867323`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 504298`; `facts.outcome.durationMs: 514729` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9335 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27375 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.88 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 4521 + 1306 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 504298`\n- `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3545,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.1648848657867323`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 504298`\n- `facts.outcome.durationMs: 514729`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 504298`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3545,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.1648848657867323`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 504298`; `facts.outcome.durationMs: 514729`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nRead the complete 3551-byte plan.md alongside brief.md and team.md. This scenario has no declared judge axes and no legacy HTML artifact, so the inline ledger records notApplicable without invented qualitative scores. Fixed composite remains 10.0.\n\nThe plan has the four requested sections, nine task rows, roster-based owners, observable completion states and the explicit movers-after-sign-off dependency. It covers seating, meeting rooms, the workshop, physical logistics and the network. The contractor participates without owning a task. Risks are concrete and assumptions are labeled.\n\nThere are advisory planning limitations beyond the passing gate. T4's completion state requires a signed layout but its dependency is only T1 (draft), not T2 (sign-off). The risks say the workshop finishes before T8 and that failed network checks postpone the move, yet T8 depends only on mover booking. Those prose/table commitments are not fully encoded in the dependency graph. The objective adds year 2026 although the supplied brief names September 30 without a year. These are review findings, not changes to the frozen score or artifact.\n\n## Positives\n\n- All six gate signals passed on the first completed artifact (sniff.latest.score: 6/6; outcome.success: true). The hard mover-booking dependency, approved owners and word ceiling are validated.\n- One batch read followed by one write produced the plan with no intervention or red flag (toolUse.byTool.read_files: 1; write_file: 1; autoAnswer.total: 0; toolUse.redFlags: []). This is efficient tool use and demonstrates delivery on a compact synthesis task.\n\n## Negatives\n\n- The two-call task still took 514729 ms, or 8m35s; output arrived at 504298 ms (outcome.durationMs; timing.timeToFirstArtifactMs). Fixed efficiency 10.0 does not mean low latency.\n- The fixed gate covers the requested structural dependency, not every inferred dependency or added date assumption (sniff.latest.signals; artifacts.otherFileCount: 4). The source-level advisory findings above qualify how ready the plan is for actual execution.\n\n## Performance and fit\n\nDecode averaged 2.88 tokens/sec over 1306 generated tokens; prompt processing averaged 162.70 tokens/sec (perf.derived). Peak VRAM was 10626/12227 MiB, process RSS 9335 MiB and total-system memory 27375/31865 MiB (perf.gpu, perf.process, perf.systemMemory). Launch records retain native 0.1.57/CUDA, 65536 context tokens, one slot and f16 KV; dense FFN of 39/64 layers remain on GPU and 25 on CPU. No explicit OOM, context overflow or unexpected load/exit failure was found. Shared GPU-memory use is not separately measured.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/de/developer/versions/1.5.1/about.md, append to the acceptance-checklist guidance: “Before finishing a plan, cross-check the dependency table against every prerequisite stated in prose; label any date or resource assumption not supplied by the user.” This is generic planning consistency, not a scenario recipe. Test on this task and an unrelated release or event plan, measuring contradictory prerequisites and unsupported assumptions while preserving output latency and all existing gates.\n\n## Strategic experiment — not applied\n\nPrototype a reusable plan representation in a proposed packages/service/src/tasks/plan-consistency.ts with task IDs, explicit prerequisites, completion criteria and assumption provenance. Render task tables and review checklists from that shared representation; use normal task/store authority for any persisted data. Structural validation can check references and cycles, while a separate review compares prose claims with declared prerequisites. The source-level T4/T8 discrepancies motivate this experiment; structural validation alone cannot infer every semantic dependency. Do not modify current evaluation gates to award extra credit.\n\n## Generalization and next experiment\n\nThis compact first-write success contrasts with incident-postmortem's unfinished large write at similar decode speed. Symptom-debug likewise used a small targeted deliverable successfully, and ops-runbook-anomaly succeeded through incremental records. The evidence supports investigating artifact size, reasoning allocation and consistency review across task shapes, rather than equating GPU fit with complete task competence. Next experiment: generic dependency/assumption review on two unrelated plans, measuring actual consistency, fixed acceptance and first-output time in a separate baseline.\n\n## Last three same-scenario trials\n\n- [plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h/facts.json): pass, fixed 10, 30.1 seconds, recorded decode 111.68 tokens/sec.\n- [plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m/facts.json): pass, fixed 10, 30.3 seconds, recorded decode 113.94 tokens/sec.\n- [plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-19-42-705Z-ytri/facts.json): pass, fixed 10, 30.2 seconds, recorded decode 100.95 tokens/sec.\n\nThese latest historical entries are from the productivity suite of the Gemma4 12B/native 0.1.54 campaign. They run the same scenario and all scored 10.0, but model, release and execution conditions differ. The current pass retains that fixed score at much higher latency; this is historical context, not a controlled regression or a three-trial reliability estimate.\n" }, { "cellId": "r1-saluki-cuda-core-conflict-synthesis", "baselineCellId": "r1-saluki-cuda-core-conflict-synthesis", "suite": "core", "scenario": "conflict-synthesis", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T05:17:18.557Z", "finishedAt": "2026-11-04T06:12:19.166Z", "durationMs": 3300609, "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1416247 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 2635, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.8810791922688621 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 31 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1416247 }, { "source": "facts", "path": "outcome.durationMs", "value": 3300609 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 555, "peakRssMb": 14646, "peakCpuPercent": 1323.7, "firstAtMs": 1, "lastAtMs": 3287950 }, "gpu": { "available": true, "sampleCount": 555, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 555, "peakUsedMb": 31301, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 162004, "totalOutputTokens": 6300, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 45311, "todayTokensCached": 18566, "todayTokensOut": 2148, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 45311, "totalTokensCached": 18566, "totalTokensOut": 2148, "totalCost": 0, "medianOutputTokensPerSec": 2.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.9, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11242, "outputTokens": 104, "cost": 0, "durationMs": 437226, "outputTokensPerSec": 2.7955766594753397, "at": "2026-11-04T06:12:11.040Z" }, "lastUpdated": "2026-11-04T06:12:11.040Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T06:12:14.394Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3289388, "meanTokensPerSec": 2.84, "genTokensPerSec": 2.84, "genTokens": 6300, "promptTokensPerSec": 165.44, "peakRssMb": 14646, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 45311, "output": 2148 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T05:17:24.531Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T05:17:24.436Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 10, "maxContextFill": 0.1430206298828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1430206298828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 3300609, "timeoutMs": 3746098, "budgetUsedFraction": 0.8810791922688621 }, "timing": { "startedAt": "2026-11-04T05:17:18.557Z", "finishedAt": "2026-11-04T06:12:19.166Z", "timeToFirstArtifactMs": 1416247, "timeToLastArtifactWriteMs": 3292464, "timeToFirstTokenMs": 86164, "firstTurnTtftMs": 74563, "timeToFirstToolCallMs": 115968 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 31, "byTool": { "read_files": 1, "write_file": 4, "read_file": 24, "replace_in_file": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 1420459, "filePath": "conflict-synthesis", "bytes": 3417, "score": 10, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-canonical-used", "claims-source-supported", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - conflict (launch DRI): a superseded value appears outside the Conflicts section; re-read memo-oldplan.md and org.md, then keep the historical value only in its reconciliation entry" }, { "atMs": 1515213, "filePath": "conflict-synthesis", "bytes": 3417, "score": 11, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - conflict (launch DRI): a superseded value appears outside the Conflicts section; re-read memo-oldplan.md and org.md, then keep the historical value only in its reconciliation entry" }, { "atMs": 2033466, "filePath": "conflict-synthesis", "bytes": 2933, "score": 10, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - conflict (launch DRI): a superseded value appears outside the Conflicts section; re-read memo-oldplan.md and org.md, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 2466988, "filePath": "conflict-synthesis", "bytes": 2650, "score": 10, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - conflict (launch DRI): a superseded value appears outside the Conflicts section; re-read memo-oldplan.md and org.md, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 2855096, "filePath": "conflict-synthesis", "bytes": 2622, "score": 12, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch DRI): a superseded value appears outside the Conflicts section; re-read memo-oldplan.md and org.md, then keep the historical value only in its reconciliation entry" }, { "atMs": 3295305, "filePath": "conflict-synthesis", "bytes": 2635, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } ], "latest": { "filePath": "conflict-synthesis", "bytes": 2635, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T05:17:18.560Z] [trial] id=conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd scenario=conflict-synthesis model=underdog-saluki-27b-iq2-mix", "[2026-11-04T05:17:18.560Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T05:17:18.560Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-LFysrL", "[2026-11-04T05:17:18.560Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "[2026-11-04T05:17:18.560Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T05:17:18.563Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T05:17:18.566Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T05:17:18.572Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-LFysrL", "[2026-11-04T05:17:18.573Z] [trial] throughput-scaled ceiling: 30m → 62m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T05:17:24.423Z] [trial] daemon spawned pid=48140 port=https://127.0.0.1:57575", "[2026-11-04T05:17:25.934Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T05:17:25.953Z] [trial] running scenario setup hook", "[2026-11-04T05:17:25.953Z] [trial] meester=zephyr", "[2026-11-04T05:17:26.440Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T05:17:26.438Z] [trial] scenario setup complete", "[2026-11-04T05:17:26.529Z] [poll] initial digests hard=ef2f7818dfc2a876 soft=0c97103f8a3471b7", "[2026-11-04T05:17:26.662Z] [poll] hard-progress: ef2f7818dfc2a876 → bfd8bb5ffe3657f5 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T05:19:19.201Z] [poll] hard-progress: bfd8bb5ffe3657f5 → 44927206b9bb981a after 113s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=74,stream=116) sniff=0/0B)", "[2026-11-04T05:32:48.737Z] [poll] hard-progress: 44927206b9bb981a → e5a960d2a3f7bfb8 after 810s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=2229,stream=3198) sniff=0/0B)", "[2026-11-04T05:33:36.686Z] [poll] hard-progress: e5a960d2a3f7bfb8 → 32d6ba6d209c05f3 after 48s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=2249,stream=3227) sniff=0/0B)", "[2026-11-04T05:40:59.136Z] [poll] hard-progress: 32d6ba6d209c05f3 → 2352c23f4dc27e10 after 442s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=3382,stream=5322) sniff=10/3417B)", "[2026-11-04T05:42:33.812Z] [poll] hard-progress: 2352c23f4dc27e10 → 389c3370598fb728 after 95s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=5,slot=3627,stream=5566) sniff=11/3417B)", "[2026-11-04T05:43:15.207Z] [poll] hard-progress: 389c3370598fb728 → 19ba303377bd8a4f after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=6,slot=3643,stream=5582) sniff=11/3417B)", "[2026-11-04T05:43:50.965Z] [poll] hard-progress: 19ba303377bd8a4f → b50796fd0f40300c after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=7,slot=3659,stream=5600) sniff=11/3417B)", "[2026-11-04T05:44:28.679Z] [poll] hard-progress: b50796fd0f40300c → d0268b6a54454036 after 38s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=8,slot=3673,stream=5614) sniff=11/3417B)" ] }, "metrics": { "process": { "sampleCount": 555, "peakRssMb": 14646, "peakCpuPercent": 1323.7, "firstAtMs": 1, "lastAtMs": 3287950 }, "gpu": { "available": true, "sampleCount": 555, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 555, "peakUsedMb": 31301, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 162004, "totalOutputTokens": 6300, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 45311, "todayTokensCached": 18566, "todayTokensOut": 2148, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 45311, "totalTokensCached": 18566, "totalTokensOut": 2148, "totalCost": 0, "medianOutputTokensPerSec": 2.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.9, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11242, "outputTokens": 104, "cost": 0, "durationMs": 437226, "outputTokensPerSec": 2.7955766594753397, "at": "2026-11-04T06:12:11.040Z" }, "lastUpdated": "2026-11-04T06:12:11.040Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T06:12:14.394Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3289388, "meanTokensPerSec": 2.84, "genTokensPerSec": 2.84, "genTokens": 6300, "promptTokensPerSec": 165.44, "peakRssMb": 14646, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 45311, "output": 2148 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T05:17:18.557Z", "finishedAt": "2026-11-04T06:12:19.166Z", "timeToFirstArtifactMs": 1416247, "timeToLastArtifactWriteMs": 3292464, "timeToFirstTokenMs": 86164, "firstTurnTtftMs": 74563, "timeToFirstToolCallMs": 115968 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T05:17:24.531Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T05:17:24.436Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 10, "maxContextFill": 0.1430206298828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1430206298828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-core-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "scenarioId": "conflict-synthesis", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/workspace/skylark-launch-synthesis/synthesis.md", "artifactBytes": 2637, "reason": "No declared judge axes or legacy HTML output. Full artifact and source evidence reviewed in postmortem, including the failed trial; no scores invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:15.134Z", "reviewId": "011", "cellId": "r1-saluki-cuda-core-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "scenarioId": "conflict-synthesis", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd", "baselineCellId": "r1-saluki-cuda-core-conflict-synthesis", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "3493b9e59b4665d11eb6e5d54d441f73bb14d1d5b44cb58c5eb2b67a9186f1b3", "evidenceSha256": "b44e6b374858c727ea51dc9c4ef08fd353df6c0eb85e7716112bc8699eebebcc", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/workspace/skylark-launch-synthesis/synthesis.md", "bytes": 2637, "sha256": "9804e60b3abdb16414ad91af6dff919b1bc3885adf47d01cece6d6e7a8e3bc19", "shownBytes": 2637, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. Qualitatively, the synthesis has the four required sections in order. Three conflicts (date, budget, DRI) quote both values with filenames and a rationale. Outside that section, only winning values appear. It cites files well over six times. Citations are numeric [n] with a source key.", "limitations": "Source files were not shown, so conflict completeness and value accuracy can't be verified. Other conflicts, such as campaign lead or engineering lead names, may exist. The brief is short and thin on plan detail. Reviewed from the artifact only.", "meanScore": null, "durationMs": 14842, "costUsd": 0.013892, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2702, "cache_read_input_tokens": 700, "output_tokens": 301, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2702, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 301, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2702, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2702 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 301, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2702, "webSearchRequests": 0, "costUSD": 0.013892, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2702, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - conflict-synthesis / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 55m 1s; budget used: 88.1%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1416247`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2635,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.8810791922688621`; `facts.toolUse.totalToolCalls: 31`; `facts.timing.timeToFirstArtifactMs: 1416247`; `facts.outcome.durationMs: 3300609` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14646 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31301 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.84 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 162004 + 6300 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 14% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1416247`\n- `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2635,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.8810791922688621`\n- `facts.toolUse.totalToolCalls: 31`\n- `facts.timing.timeToFirstArtifactMs: 1416247`\n- `facts.outcome.durationMs: 3300609`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1416247`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2635,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.8810791922688621`; `facts.toolUse.totalToolCalls: 31`; `facts.timing.timeToFirstArtifactMs: 1416247`; `facts.outcome.durationMs: 3300609`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nRead the complete 2637-byte synthesis and all six supplied source/style files. No judge axes are declared, so the ledger records notApplicable without invented scores. Fixed composite remains 8.5. The final brief correctly selects September 1, EUR 210000 and Marcus, explains each competing source and precedence rule, and confines superseded specific values to the conflicts section. Its numbered citations resolve to named sources, and the open questions distinguish missing evidence from decisions.\n\n## Positives\n\n- All 13 acceptance signals passed, including source-read grounding and all three reconciliations (sniff.latest.score: 13/13; outcome.success: true). The final artifact is materially grounded in the provided evidence.\n- Bounded feedback eventually corrected earlier stale values and unsupported amounts without user intervention (autoAnswer.total: 0; sniff.latest.failReason: null; log.txt progression). No expected role was missing and no fixed-rubric red flag fired (team.missingExpectedRoles: []; toolUse.redFlags: []).\n\n## Negatives\n\n- Delivery took 55m1s and 31 tool calls, including 24 individual reads, four writes and two replacements (outcome.durationMs: 3300609; toolUse.byTool). The severe call-count dimension explains the fixed efficiency penalty, despite final success.\n- First output took 1416247 ms and final repair arrived at 3292464 ms (timing.timeToFirstArtifactMs; timeToLastArtifactWriteMs). The report repeatedly needed corrections to keep historical claims out of the current plan; clean final content does not establish first-pass reliability.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/re/researcher/versions/1.2.0/about.md, add: “For conflicting sources, record the chosen value, its authority and the displaced value before drafting. Keep displaced claims only in the reconciliation section; check the rest of the document against the chosen values after editing.” Test this generic source-precedence workflow on unrelated policy and project-brief synthesis, measuring first-pass groundedness and repair calls. Do not add the current fixture's names, dates or amounts to prompts.\n\n## Strategic experiment — not applied\n\nPrototype a revision-scoped claim ledger in a proposed packages/core/src/local-loop/source-claim-ledger.ts that records source, authority and current/superseded status alongside each extracted claim. Expose it to drafting/review through existing tool boundaries and invalidate it when source revisions change. It should support review across sections without repeatedly rereading unchanged sources. The 24 individual reads and cross-section stale-value repairs motivate this experiment; a ledger would not itself prove natural-language correctness. Validate on a different source-conflict domain and preserve all existing gates.\n\n## Generalization and next experiment\n\nThis task eventually succeeded where the longer incident report never committed its first artifact. Both used the Researcher role at about 2.8 tokens/sec, so role availability and successful GPU loading do not establish synthesis reliability. Ops-runbook-anomaly also needed record-format repairs but finished via short writes. Next experiment: the generic source-precedence ledger wording across two synthesis domains, with unchanged sampling and acceptance gates, measuring groundedness, reads and time.\n\n## Performance and GPU fit\n\nDecode 2.84 tokens/sec over 6300 completed-request generated tokens; prompt processing 165.44 tokens/sec. Peak VRAM 10642/12227 MiB, RSS 14646 MiB, whole-system memory 31301/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57/CUDA launch retained 65536 context tokens, one slot, f16 KV and 25 CPU FFN layers (39/64 dense FFN layers on GPU). No explicit OOM, context overflow or unexpected load/exit failure was found. Shared GPU-memory use is not separately measured.\n\n## Last three same-scenario trials\n\n- [conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2/facts.json): pass, fixed 10, 36.0 seconds, recorded decode 122.83 tokens/sec.\n- [conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p/facts.json): pass, fixed 9.5, 91.5 seconds, recorded decode 119.86 tokens/sec.\n- [conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-37-13-705Z-n5kt/facts.json): pass, fixed 9.5, 130.1 seconds, recorded decode 127.78 tokens/sec.\n\nHistorical context only: these productivity entries used Gemma4 12B/native 0.1.54. They are not the current frozen comparison identity.\n" }, { "cellId": "r1-saluki-cuda-productivity-constrained-comms", "baselineCellId": "r1-saluki-cuda-productivity-constrained-comms", "suite": "productivity", "scenario": "constrained-comms", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T06:15:01.379Z", "finishedAt": "2026-11-04T06:23:01.424Z", "durationMs": 480045, "success": false, "reason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "constrained-comms", "score": 8, "bytes": 697, "failReason": "word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 171710 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 697, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words" } } ] }, "quality": { "score": 5, "ruleId": "Q_PARTIAL_UNKNOWN_MAX", "summary": "An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 171710 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 697, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.19221798359012718 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 7 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 171710 }, { "source": "facts", "path": "outcome.durationMs", "value": 480045 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6, "band": "needs-tuning" }, "facts": { "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 80, "peakRssMb": 13236, "peakCpuPercent": 1200.9, "firstAtMs": 0, "lastAtMs": 468394 }, "gpu": { "available": true, "sampleCount": 80, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 80, "peakUsedMb": 29764, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26337, "totalOutputTokens": 839, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 20419, "todayTokensCached": 11507, "todayTokensOut": 725, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 20419, "totalTokensCached": 11507, "totalTokensOut": 725, "totalCost": 0, "medianOutputTokensPerSec": 3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 3, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5159, "outputTokens": 196, "cost": 0, "durationMs": 99047, "outputTokensPerSec": 2.972329124876037, "at": "2026-11-04T06:22:55.561Z" }, "lastUpdated": "2026-11-04T06:22:55.561Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T06:22:57.579Z" } }, "source": "engine-log" }, "derived": { "durationMs": 469724, "meanTokensPerSec": 2.99, "genTokensPerSec": 2.99, "genTokens": 839, "promptTokensPerSec": 173.69, "peakRssMb": 13236, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 20419, "output": 725 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T06:15:07.821Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T06:15:07.714Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wren": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 8, "maxContextFill": 0.093719482421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.093719482421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words", "durationMs": 480045, "timeoutMs": 2497399, "budgetUsedFraction": 0.19221798359012718 }, "timing": { "startedAt": "2026-11-04T06:15:01.379Z", "finishedAt": "2026-11-04T06:23:01.424Z", "timeToFirstArtifactMs": 171710, "timeToLastArtifactWriteMs": 474182, "timeToFirstTokenMs": 81496, "firstTurnTtftMs": 69932, "timeToFirstToolCallMs": 99006 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 7, "byTool": { "read_files": 1, "write_file": 3, "read_file": 2, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 173358, "filePath": "constrained-comms", "bytes": 538, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 82 words, need ≥ 140 — the notice must be 140-220 words" }, { "atMs": 286251, "filePath": "constrained-comms", "bytes": 656, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 105 words, need ≥ 140 — the notice must be 140-220 words" }, { "atMs": 375008, "filePath": "constrained-comms", "bytes": 697, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words" } ], "latest": { "filePath": "constrained-comms", "bytes": 697, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T06:15:01.384Z] [trial] id=constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b scenario=constrained-comms model=underdog-saluki-27b-iq2-mix", "[2026-11-04T06:15:01.384Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "[2026-11-04T06:15:01.384Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-constrained-comms-Eqjwll", "[2026-11-04T06:15:01.384Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T06:15:01.384Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T06:15:01.388Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T06:15:01.392Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T06:15:01.399Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-constrained-comms-Eqjwll", "[2026-11-04T06:15:01.399Z] [trial] throughput-scaled ceiling: 20m → 42m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T06:15:07.808Z] [trial] daemon spawned pid=20204 port=https://127.0.0.1:64028", "[2026-11-04T06:15:08.745Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T06:15:08.764Z] [trial] meester=imara", "[2026-11-04T06:15:08.764Z] [trial] running scenario setup hook", "[2026-11-04T06:15:09.232Z] [trial] scenario setup complete", "[2026-11-04T06:15:09.233Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T06:15:09.308Z] [poll] initial digests hard=8c15e6d6cc0fc86b soft=601add4ab7b79253", "[2026-11-04T06:15:09.402Z] [poll] hard-progress: 8c15e6d6cc0fc86b → c8a3a8c9d437bdb2 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T06:16:44.116Z] [poll] hard-progress: c8a3a8c9d437bdb2 → 8addc12c0f27aeb4 after 95s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=50,stream=75) sniff=0/0B)", "[2026-11-04T06:17:54.869Z] [poll] hard-progress: 8addc12c0f27aeb4 → 5ddfb5ca8feada1c after 71s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=189,stream=341) sniff=8/538B)", "[2026-11-04T06:18:42.401Z] [poll] hard-progress: 5ddfb5ca8feada1c → a6ae2cb1adaceb15 after 48s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=3,slot=206,stream=360) sniff=8/538B)", "[2026-11-04T06:19:47.759Z] [poll] hard-progress: a6ae2cb1adaceb15 → 7b271e567afd02f4 after 65s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=4,slot=368,stream=680) sniff=8/656B)", "[2026-11-04T06:20:11.549Z] [poll] hard-progress: 7b271e567afd02f4 → 4572eb3392c7effb after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=5,slot=383,stream=699) sniff=8/656B)", "[2026-11-04T06:21:16.510Z] [poll] hard-progress: 4572eb3392c7effb → d29365dc73cda1fb after 65s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=6,slot=525,stream=980) sniff=8/697B)", "[2026-11-04T06:22:57.104Z] [poll] terminal (scenario handoff): repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 112 words, need ≥ 140 — the notice must be 140-220 words", "[2026-11-04T06:22:57.616Z] [capture] dumped 1 session(s)" ] }, "metrics": { "process": { "sampleCount": 80, "peakRssMb": 13236, "peakCpuPercent": 1200.9, "firstAtMs": 0, "lastAtMs": 468394 }, "gpu": { "available": true, "sampleCount": 80, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 80, "peakUsedMb": 29764, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26337, "totalOutputTokens": 839, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 20419, "todayTokensCached": 11507, "todayTokensOut": 725, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 20419, "totalTokensCached": 11507, "totalTokensOut": 725, "totalCost": 0, "medianOutputTokensPerSec": 3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 3, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5159, "outputTokens": 196, "cost": 0, "durationMs": 99047, "outputTokensPerSec": 2.972329124876037, "at": "2026-11-04T06:22:55.561Z" }, "lastUpdated": "2026-11-04T06:22:55.561Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T06:22:57.579Z" } }, "source": "engine-log" }, "derived": { "durationMs": 469724, "meanTokensPerSec": 2.99, "genTokensPerSec": 2.99, "genTokens": 839, "promptTokensPerSec": 173.69, "peakRssMb": 13236, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 20419, "output": 725 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T06:15:01.379Z", "finishedAt": "2026-11-04T06:23:01.424Z", "timeToFirstArtifactMs": 171710, "timeToLastArtifactWriteMs": 474182, "timeToFirstTokenMs": 81496, "firstTurnTtftMs": 69932, "timeToFirstToolCallMs": 99006 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T06:15:07.821Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T06:15:07.714Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wren": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 8, "maxContextFill": 0.093719482421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.093719482421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-constrained-comms", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "scenarioId": "constrained-comms", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/workspace/driftwater-outage-notice/customer-notice.md", "artifactBytes": 701, "reason": "No declared judge axes or legacy HTML output. Full artifact and source evidence reviewed in postmortem, including the failed trial; no scores invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:21.043Z", "reviewId": "012", "cellId": "r1-saluki-cuda-productivity-constrained-comms", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "scenarioId": "constrained-comms", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "baselineCellId": "r1-saluki-cuda-productivity-constrained-comms", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "24e75a518588c97ca582534fbdc8b17ea863f4cbe4930b02cbe9f22b0ad7a07d", "evidenceSha256": "66893a8bdff5e7e8aeecbc5e7a693498bc215d095e01382ed29a562e0ec26b88", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/workspace/driftwater-outage-notice/customer-notice.md", "bytes": 701, "sha256": "9b3e78c5c6b3288a92ecbcaa1470e2964c31cd783b12d4b6b9d42bea2a40db57", "shownBytes": 701, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b\\workspace\\driftwater-outage-notice\\facts\\incident-brief.md", "bytes": 524, "sha256": "a65cd16d09c63d27126363a03f1bc664d26e4c6fc58eae192477d98ac3d02251", "shownBytes": 524, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b\\workspace\\driftwater-outage-notice\\facts\\legal-requirements.md", "bytes": 446, "sha256": "f15d48f9a7a37a2e3d855566ffd9cbdb9f0633a9f8c7f0756816b40a15815920", "shownBytes": 446, "truncated": false } ], "scoreAxes": {}, "justification": "The notice has the June 30 window, 38 minutes, the exact sentence, the support email and the one-month credit. Facts match the brief, and no banned phrases appear. But my manual count is about 105–110 words including the title, below the required 140–220. That breaks a hard constraint.", "limitations": "The word count is manual and approximate, with no tool run. The transcript isn't shown, so I can't confirm the model read the files or that write_file succeeded. I reviewed only the final artifact text. No numeric axes were declared.", "meanScore": null, "durationMs": 18398, "costUsd": 0.019446, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2448, "cache_read_input_tokens": 700, "output_tokens": 958, "output_tokens_details": { "thinking_tokens": 671 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2448, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 958, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2448, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2448 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 958, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2448, "webSearchRequests": 0, "costUSD": 0.019446, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 671, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2448, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - constrained-comms / underdog-saluki-27b-iq2-mix\n\n**Composite: 6.0 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 8m 0s; budget used: 19.2%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`; `facts.timing.timeToFirstArtifactMs: 171710`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":697,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...` |\n| Output quality (25%) | 5 / 10 | An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 171710`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":697,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.19221798359012718`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 171710`; `facts.outcome.durationMs: 480045` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **6.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13236 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29764 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.99 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 26337 + 839 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 9% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`\n- `facts.timing.timeToFirstArtifactMs: 171710`\n- `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":697,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n\n### Output quality - Q_PARTIAL_UNKNOWN_MAX\n\nAn authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 171710`\n- `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":697,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.19221798359012718`\n- `facts.toolUse.totalToolCalls: 7`\n- `facts.timing.timeToFirstArtifactMs: 171710`\n- `facts.outcome.durationMs: 480045`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`; `facts.timing.timeToFirstArtifactMs: 171710`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":697,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n- Quality: An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 171710`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":697,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.19221798359012718`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 171710`; `facts.outcome.durationMs: 480045`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nThe 701-byte customer notice exists and was read alongside all three fact/requirement files and the stale draft. It has no declared judge axes, so record notApplicable, not noArtifact, and do not invent axis scores. Fixed composite remains 6.0 and the failed verdict remains unchanged.\n\nThe notice uses the correct 38-minute outage, 12% scope, payment-data disclosure, support address and one-month automatic credit. It avoids the explicitly banned phrases. It is only 112 words against the required 140–220. Two additional source-level issues deserve review: “all other parts of the site” and no other service being interrupted are broader than the source's statement that browsing was unaffected; “so the same failure does not recur” implies a stronger future assurance than the evidence establishes. These advisory findings are separate from the gate, which marked its grounded-facts signal present.\n\n## Positives\n\n- Eight recorded signals passed for required disclosures, credit, banned-phrase avoidance and the gate's factual checks (sniff.latest.score: 8; signals). The stale draft's 83-minute duration was not copied.\n- The expected Copywriter role delivered a real file after a batch read, without user intervention or tool red flags (team.missingExpectedRoles: []; toolUse.byTool.read_files: 1; timing.timeToFirstArtifactMs: 171710; autoAnswer.total: 0; toolUse.redFlags: []).\n\n## Negatives\n\n- Four delivered-and-completed post-nudge attempts exhausted the unchanged repair budget with the notice still below its word floor (outcome.reason; sniff.latest.failReason: 112 words, need at least 140). This was a content-constraint failure, not OOM or lack of output.\n- The log shows growth to 105 then 112 words followed by a byte-identical attempted repair at 06:22:57Z (toolUse.byTool.write_file: 3; replace_in_file: 1; log.txt). Repeating a completed write did not satisfy the clearly stated remaining requirement. Total duration was 480045 ms, about eight minutes.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/co/copywriter/versions/1.1.0/about.md, add: “When a word range is given, draft toward its midpoint and verify both limits. If short, expand only confirmed cause, impact, remedies or practical next steps; do not invent reassuring claims.” This generic instruction targets numeric constraint following without encoding this scenario's word band. Validate on another length-constrained notice and an unrelated summary; compare compliance and unsupported claims with unchanged gates.\n\n## Strategic experiment — not applied\n\nExtend packages/mcp/src/workspace-write-quality.ts with optional task-supplied word-band metadata and deterministic candidate counts, returning the measured count and deficit/excess with write feedback. Keep the feature advisory to ordinary authoring and do not silently pad or fabricate content. Pair it with a generic no-op repair diagnostic in packages/core/src/local-loop/file-repair-policy.ts when a supposedly repaired file is byte-identical. The repeated small/identical revisions show why exact measurement is more actionable than another unmeasured rewrite. Test these tools on different prose tasks in a separate campaign.\n\n## Generalization and historical interpretation\n\nPlan-and-estimate met its word ceiling and content structure on one write, but a ceiling-only pass does not show competence with a lower bound. Conflict-synthesis required many successful repairs; this notice exhausted its repairs before satisfying a simple numeric constraint. Next experiment: midpoint-and-count guidance on two unrelated bounded-writing tasks, measuring both size compliance and evidence fidelity. The three older notice trials passed, but different model/engine conditions and only one current trial prevent a controlled reliability comparison.\n\n## Performance and GPU fit\n\nDecode 2.99 tokens/sec over 839 completed-request generated tokens; prompt processing 173.69 tokens/sec. Peak VRAM 10626/12227 MiB, RSS 13236 MiB, whole-system memory 29764/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57/CUDA launch retained 65536 context tokens, one slot, f16 KV and 25 CPU FFN layers (39/64 dense FFN layers on GPU). No explicit OOM, context overflow or unexpected load/exit failure was found. Shared GPU-memory use is not separately measured.\n\n## Last three same-scenario trials\n\n- [constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/facts.json): pass, fixed 10, 41.3 seconds, recorded decode 136.68 tokens/sec.\n- [constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8/facts.json): pass, fixed 10, 29.1 seconds, recorded decode 127.21 tokens/sec.\n- [constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-20-815Z-u9t4/facts.json): pass, fixed 10, 29.1 seconds, recorded decode 142.22 tokens/sec.\n\nHistorical context only: these productivity entries used Gemma4 12B/native 0.1.54. They are not the current frozen comparison identity.\n" }, { "cellId": "r1-saluki-cuda-productivity-records-intake", "baselineCellId": "r1-saluki-cuda-productivity-records-intake", "suite": "productivity", "scenario": "records-intake", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T06:25:47.186Z", "finishedAt": "2026-11-04T07:18:07.881Z", "durationMs": 3140695, "success": false, "reason": "hit hard ceiling (3121748ms) — last hard progress 1525s before the end; the deliverable never closed", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "failureMode": "timeout", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "records-intake", "score": 0, "bytes": 0 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "timeout" }, { "source": "facts", "path": "outcome.reason", "value": "hit hard ceiling (3121748ms) — last hard progress 1525s before the end; the deliverable never closed" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "quality": { "score": 0, "ruleId": "Q_NO_PRODUCED_OUTPUT", "summary": "No authored artifact event or scored sniff output was observed.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 6 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 3140695 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 2, "band": "framework-gap" }, "facts": { "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 530, "peakRssMb": 9725, "peakCpuPercent": 1210.9, "firstAtMs": 1, "lastAtMs": 3127470 }, "gpu": { "available": true, "sampleCount": 530, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 530, "peakUsedMb": 27504, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 5600, "totalOutputTokens": 4326, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T07:18:02.564Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3128979, "meanTokensPerSec": 2.8, "genTokensPerSec": 2.8, "genTokens": 4326, "promptTokensPerSec": 162.07, "peakRssMb": 9725, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T06:25:53.088Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T06:25:52.987Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "griet": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0550689697265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0550689697265625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "timeout", "reason": "hit hard ceiling (3121748ms) — last hard progress 1525s before the end; the deliverable never closed", "durationMs": 3140695, "timeoutMs": 3121748, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-04T06:25:47.186Z", "finishedAt": "2026-11-04T07:18:07.881Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 47471, "firstTurnTtftMs": 36003, "timeToFirstToolCallMs": 107868 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 6, "byTool": { "list_dir": 1, "read_file": 4, "read_files": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [], "latest": null }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T06:25:47.191Z] [trial] id=records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry scenario=records-intake model=underdog-saluki-27b-iq2-mix", "[2026-11-04T06:25:47.191Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "[2026-11-04T06:25:47.191Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T06:25:47.191Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T06:25:47.191Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-records-intake-UHFcM2", "[2026-11-04T06:25:47.194Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T06:25:47.198Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T06:25:47.204Z] [trial] throughput-scaled ceiling: 25m → 52m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T06:25:47.204Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-records-intake-UHFcM2", "[2026-11-04T06:25:53.047Z] [trial] daemon spawned pid=45044 port=https://127.0.0.1:52374", "[2026-11-04T06:25:54.555Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T06:25:54.573Z] [trial] meester=bastien", "[2026-11-04T06:25:54.573Z] [trial] running scenario setup hook", "[2026-11-04T06:25:55.017Z] [trial] scenario setup complete", "[2026-11-04T06:25:55.019Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T06:25:55.096Z] [poll] initial digests hard=1e93b22ffe59f263 soft=b11e3bfec791b70d", "[2026-11-04T06:25:55.168Z] [poll] hard-progress: 1e93b22ffe59f263 → 330e2dbb7e13a74d after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T06:27:35.855Z] [poll] hard-progress: 330e2dbb7e13a74d → 0c5fd055d3271084 after 101s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=82,stream=114) sniff=0/0B)", "[2026-11-04T06:52:37.155Z] [poll] hard-progress: 0c5fd055d3271084 → 11f3dadecd2f4883 after 1501s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=4136,stream=4196) sniff=0/0B)", "[2026-11-04T07:18:02.016Z] [poll] hard ceiling reached after 3121748ms", "[2026-11-04T07:18:02.656Z] [capture] dumped 1 session(s)", "[2026-11-04T07:18:02.659Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T07:18:02.660Z] [capture] INCOMPLETE TRANSCRIPT for 1 session(s) — griet: 0 persisted vs 6 recorded. Assistant turns commit at turn END, so a mid-turn termination loses them. Triage from session-telemetry.json and daemon.log, NOT from the empty transcript.", "[2026-11-04T07:18:02.719Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T07:18:07.880Z] [trial] daemon shut down" ] }, "metrics": { "process": { "sampleCount": 530, "peakRssMb": 9725, "peakCpuPercent": 1210.9, "firstAtMs": 1, "lastAtMs": 3127470 }, "gpu": { "available": true, "sampleCount": 530, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 530, "peakUsedMb": 27504, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 5600, "totalOutputTokens": 4326, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T07:18:02.564Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3128979, "meanTokensPerSec": 2.8, "genTokensPerSec": 2.8, "genTokens": 4326, "promptTokensPerSec": 162.07, "peakRssMb": 9725, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T06:25:47.186Z", "finishedAt": "2026-11-04T07:18:07.881Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 47471, "firstTurnTtftMs": 36003, "timeToFirstToolCallMs": 107868 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T06:25:53.088Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T06:25:52.987Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "griet": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0550689697265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0550689697265625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-records-intake", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "scenarioId": "records-intake", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/workspace/conference-registration-intake/records/attendees.csv", "reason": "Required CSV was not produced before the absolute timeout; no scores invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:18.274Z", "reviewId": "013", "cellId": "r1-saluki-cuda-productivity-records-intake", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "scenarioId": "records-intake", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "baselineCellId": "r1-saluki-cuda-productivity-records-intake", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "0972b069199c3ee08145c92f76aaa9ac5e726d27e87359f1988aa98b4f35c32c", "evidenceSha256": "b13e64f256b5050a0246ca18e99a87b4b20542506bda1fad39cd912d2380721d", "artifactFiles": [], "contextFiles": [], "scoreAxes": {}, "justification": "The declared deliverable, records/attendees.csv in the conference-registration-intake workspace, was missing at closeout. The evidence has no artifacts, no context files and no transcript. The run ended as runner-terminal, so the cause is unknown. Nothing shows whether a script ran or whether the run was interrupted, so no quality judgment can be made.", "limitations": "Only the missing-path record and run metadata were available. There was no execution log, no source file contents and no partial output. I can't tell whether the run failed, was interrupted or was never completed. I assigned no scores and made no model-failure or success claim.", "meanScore": null, "durationMs": 14633, "costUsd": 0.008966, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1438, "cache_read_input_tokens": 700, "output_tokens": 314, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1438, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 314, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1438, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1438 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 314, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1438, "webSearchRequests": 0, "costUSD": 0.008966, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1438, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - records-intake / underdog-saluki-27b-iq2-mix\n\n**Composite: 2.0 / 10** (band: framework-gap)\n**Outcome: timeout** (duration: 52m 21s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (3121748ms) — last hard progress 1525s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null` |\n| Output quality (25%) | 0 / 10 | No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 6`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 3140695` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 0 + 0.2 * 2.5 + 0.15 * 10 = **2.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9725 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27504 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.8 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 5600 + 4326 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"timeout\"`\n- `facts.outcome.reason: \"hit hard ceiling (3121748ms) — last hard progress 1525s before the end; the deliverable never closed\"`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: null`\n\n### Output quality - Q_NO_PRODUCED_OUTPUT\n\nNo authored artifact event or scored sniff output was observed.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: null`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 6`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 3140695`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (3121748ms) — last hard progress 1525s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null`\n- Quality: No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 6`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 3140695`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory review and outcome\n\nThe required records/attendees.csv is absent. Record noArtifact without qualitative scores; the five remaining workspace files are seeded inputs/library material. The fixed timeout verdict and composite 2.0 are unchanged. No output was completed in 52m21s (outcome.durationMs: 3140695; timing.timeToFirstArtifactMs: null).\n\n## Positives\n\n- Source access worked: the first tool batch listed the workspace and read the schema plus all three intake sources (toolUse.byTool.list_dir: 1; read_file: 4; recording/chat-events.jsonl at 06:27:35Z). There was no missing-file infrastructure error.\n- The engine stayed alive and generating with no tool red flag or user intervention (toolUse.redFlags: []; autoAnswer.total: 0; session-telemetry.json inflight: true). Loading and source access succeeded even though delivery failed.\n\n## Negatives\n\n- The completed calls were entirely reads/listing; a later read_files call reread the same four inputs after roughly 25 minutes of generation (toolUse.totalToolCalls: 6; byTool; recording/chat-events.jsonl at 06:52:31Z). No write_file completed or began in the captured tool-argument stream.\n- The unchanged absolute ceiling ended the task 1525 seconds after its last hard progress (outcome.failureMode: timeout; outcome.reason). The ceiling was scaled using preflight 9.61 tokens/sec, while this run decoded around 2.8. This observation does not authorize changing the frozen timeout or retrying the failure.\n- Completed-request metrics report 4326 output tokens, but the unfinished request reached 4207 additional request-local tokens in the live phase record. Do not treat the fixed total as all generation. The persisted session transcript is explicitly incomplete after mid-turn termination (perf.derived; session-telemetry.json; log.txt; advisory-generation-evidence.json).\n\n## Tactical experiment — not applied\n\nFor a separate A/B, set tuning.reasoning.thinkingBudget and the effective thinking-general/thinking-coding override from 4096 to 1024 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json. Preserve sampling, output caps and deadlines. Two long reasoning periods followed by rereads make this a concrete delivery-latency hypothesis, not a proven fix. Measure completed output and correctness on intake, data transformation and an unrelated coding regression.\n\n## Strategic experiment — not applied\n\nIn packages/core/src/local-loop/deliverable-read-pacing.ts, investigate carrying an explicit file-deliverable intent with required-source coverage and revision-aware read evidence across provider iterations. Once all unchanged inputs are covered, a repeated full read should produce a bounded write-now recovery rather than restarting the same discovery phase. Do not simply lower read-count thresholds: first establish why the current intent/pacing tracker did not lead to a mutation. Compare another data task and a legitimate multi-file code investigation to avoid blocking necessary reads.\n\n## Generalization and next experiment\n\nIncident-postmortem similarly remained alive without committing output; data-wrangle generated a helper but not the requested dataset. In contrast, meeting-followup produced an exact four-row CSV at essentially the same decode speed. The reusable failure mode is delayed first delivery and source rereading, not inability to load or universally emit CSV. Test the smaller reasoning allowance across those task shapes in a separate baseline, keeping this failure intact.\n\n## Performance and fit\n\nRecorded decode 2.8 tokens/sec; prefill 162.07. Peak VRAM 10626/12227 MiB, process RSS 9725 MiB, whole-system use 27504/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Launch remains native 0.1.57/CUDA, 65536 context tokens, one slot, f16 KV and 25 CPU FFN layers (39/64 dense FFN layers on GPU). No explicit OOM, context overflow or unexpected native exit found. Shared GPU memory is not separately measured.\n\n## Last three same-scenario trials\n\n- [records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/facts.json): pass, fixed 10, 129.7 seconds, recorded decode 143.1 tokens/sec.\n- [records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i/facts.json): pass, fixed 10, 130.0 seconds, recorded decode 138.8 tokens/sec.\n- [records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-13-06-381Z-vwho/facts.json): pass, fixed 10, 130.4 seconds, recorded decode 137.57 tokens/sec.\n\nThe plan comparison includes the earlier Saluki core entry; older external entries use Gemma4 12B/native 0.1.54 and are historical context, not a controlled regression. No current suite entry has completed its requested three repetitions yet.\n" }, { "cellId": "r1-saluki-cuda-productivity-plan-and-estimate", "baselineCellId": "r1-saluki-cuda-productivity-plan-and-estimate", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T07:20:21.296Z", "finishedAt": "2026-11-04T07:29:20.524Z", "durationMs": 539228, "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 529838 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 3132, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.17273271257000886 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 529838 }, { "source": "facts", "path": "outcome.durationMs", "value": 539228 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 90, "peakRssMb": 9336, "peakCpuPercent": 1203.1, "firstAtMs": 0, "lastAtMs": 527208 }, "gpu": { "available": true, "sampleCount": 90, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 90, "peakUsedMb": 27138, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4522, "totalOutputTokens": 1425, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T07:29:16.427Z" } }, "source": "engine-log" }, "derived": { "durationMs": 528607, "meanTokensPerSec": 2.99, "genTokensPerSec": 2.99, "genTokens": 1425, "promptTokensPerSec": 161.74, "peakRssMb": 9336, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T07:20:27.367Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T07:20:27.262Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0569305419921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0569305419921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 539228, "timeoutMs": 3121748, "budgetUsedFraction": 0.17273271257000886 }, "timing": { "startedAt": "2026-11-04T07:20:21.296Z", "finishedAt": "2026-11-04T07:29:20.524Z", "timeToFirstArtifactMs": 529838, "timeToLastArtifactWriteMs": 529838, "timeToFirstTokenMs": 47195, "firstTurnTtftMs": 35619, "timeToFirstToolCallMs": 67892 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "read_files": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 534620, "filePath": "plan-and-estimate", "bytes": 3132, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } ], "latest": { "filePath": "plan-and-estimate", "bytes": 3132, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T07:20:21.300Z] [trial] id=plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek scenario=plan-and-estimate model=underdog-saluki-27b-iq2-mix", "[2026-11-04T07:20:21.301Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "[2026-11-04T07:20:21.301Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T07:20:21.301Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-TqVWTu", "[2026-11-04T07:20:21.301Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T07:20:21.304Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T07:20:21.308Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T07:20:21.314Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-TqVWTu", "[2026-11-04T07:20:21.314Z] [trial] throughput-scaled ceiling: 25m → 52m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T07:20:27.290Z] [trial] daemon spawned pid=17644 port=https://127.0.0.1:62050", "[2026-11-04T07:20:28.700Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T07:20:28.719Z] [trial] meester=soren", "[2026-11-04T07:20:28.719Z] [trial] running scenario setup hook", "[2026-11-04T07:20:29.177Z] [trial] scenario setup complete", "[2026-11-04T07:20:29.179Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T07:20:29.251Z] [poll] initial digests hard=b895425cbfdd74c0 soft=343cb8a54f2c49ec", "[2026-11-04T07:20:29.311Z] [poll] hard-progress: b895425cbfdd74c0 → b158b41e46db30f0 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T07:21:34.846Z] [poll] hard-progress: b158b41e46db30f0 → e9e8e29b34254d1e after 66s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=49,stream=61) sniff=0/0B)", "[2026-11-04T07:29:15.916Z] [poll] terminal: success=true reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "[2026-11-04T07:29:16.457Z] [capture] dumped 1 session(s)", "[2026-11-04T07:29:16.460Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T07:29:16.511Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T07:29:20.523Z] [trial] daemon shut down", "[2026-11-04T07:29:20.534Z] [trial] PASS duration=539228ms reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)" ] }, "metrics": { "process": { "sampleCount": 90, "peakRssMb": 9336, "peakCpuPercent": 1203.1, "firstAtMs": 0, "lastAtMs": 527208 }, "gpu": { "available": true, "sampleCount": 90, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 90, "peakUsedMb": 27138, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4522, "totalOutputTokens": 1425, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T07:29:16.427Z" } }, "source": "engine-log" }, "derived": { "durationMs": 528607, "meanTokensPerSec": 2.99, "genTokensPerSec": 2.99, "genTokens": 1425, "promptTokensPerSec": 161.74, "peakRssMb": 9336, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T07:20:21.296Z", "finishedAt": "2026-11-04T07:29:20.524Z", "timeToFirstArtifactMs": 529838, "timeToLastArtifactWriteMs": 529838, "timeToFirstTokenMs": 47195, "firstTurnTtftMs": 35619, "timeToFirstToolCallMs": 67892 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T07:20:27.367Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T07:20:27.262Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0569305419921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0569305419921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "scenarioId": "plan-and-estimate", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/workspace/harbourview-office-relocation/plan.md", "reason": "No declared judge axes or legacy HTML output; full plan and source brief reviewed in postmortem." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:30.137Z", "reviewId": "014", "cellId": "r1-saluki-cuda-productivity-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "scenarioId": "plan-and-estimate", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek", "baselineCellId": "r1-saluki-cuda-productivity-plan-and-estimate", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "50125dec6b5947ed796979371b69bda2f13353f0205e23104c56dd17f45542a7", "evidenceSha256": "4d1eb7fbf94eb2a8ed453198e5cc5e45d5192252ef17369202169edf839a1070", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/workspace/harbourview-office-relocation/plan.md", "bytes": 3136, "sha256": "fdc179aa734adb91380ca44567c2b7eb768b64f8ed51d07c7aa0100b9c028d72", "shownBytes": 3136, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek\\workspace\\harbourview-office-relocation\\brief.md", "bytes": 732, "sha256": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "shownBytes": 732, "truncated": false } ], "scoreAxes": {}, "justification": "plan.md has the four sections in order, and the table header is exact with 10 rows. Owners are all roster names and Ola owns nothing. T3 (movers) depends on T2 (sign-off), and floor plan, movers and network are all covered. Length is well under 1200 words. Weaker points: T4 depends only on T1, not the sign-off.", "limitations": "team.md was not shown, so roster names are unverified. The final reply (path plus precis) is not in the evidence. Word count is estimated. No dates or schedule are given. Sign-off needing \"all 18 approvals\" is an invented detail. Source-only review.", "meanScore": null, "durationMs": 14985, "costUsd": 0.016518, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3311, "cache_read_input_tokens": 700, "output_tokens": 320, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3311, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 320, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3311, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3311 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 320, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3311, "webSearchRequests": 0, "costUSD": 0.016518, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3311, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - plan-and-estimate / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 8m 59s; budget used: 17.3%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 529838`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3132,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.17273271257000886`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 529838`; `facts.outcome.durationMs: 539228` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9336 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27138 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.99 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 4522 + 1425 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 529838`\n- `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3132,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.17273271257000886`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 529838`\n- `facts.outcome.durationMs: 539228`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 529838`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3132,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.17273271257000886`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 529838`; `facts.outcome.durationMs: 539228`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nRead the complete 3136-byte plan against the same brief and roster used for the core entry. No judge axes are declared, so record notApplicable without invented scores. Fixed composite remains 10.0. The plan covers all workstreams with ten task rows, named internal owners, concrete completion criteria and mover booking after floor-plan sign-off.\n\nAdvisory consistency gaps remain: T8 packing requires labels based on the signed floor plan but has no prerequisite, and T9 does not depend on T8. T9 depends on the workshop build T4, while a risk mitigation says the move proceeds if the build slips; that exception is not represented in the table. T2 adds approvals from all 18 staff beyond the supplied formal-sign-off requirement. These are source-level review findings, not retroactive gate changes.\n\n## Positives\n\n- All six structural/workstream/word-ceiling signals passed with one batch read and one write (sniff.latest.score: 6/6; toolUse.byTool.read_files: 1; write_file: 1).\n- No tool red flag, missing role or user intervention was recorded (toolUse.redFlags: []; team.missingExpectedRoles: []; autoAnswer.total: 0). A second distinct suite entry has now completed this scenario at fixed 10.0.\n\n## Negatives\n\n- First output arrived at 529838 ms of a 539228 ms trial (timing.timeToFirstArtifactMs; outcome.durationMs). Few tool calls still translate into nearly nine minutes of latency.\n- The passing gate validates the explicit mover constraint, not every prose/table dependency (sniff.latest.signals; artifacts.otherFileCount: 4). The consistency issues above recur in a different form from the earlier core artifact.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/de/developer/versions/1.5.1/about.md, add: “Cross-check every task's completion criteria and every contingency against the dependency table; label added approval requirements as assumptions.” Test on two unrelated plans, measuring contradictory or missing prerequisites without changing fixed gates. This is the same broad consistency concern seen in the earlier core plan, not fixture-specific prompt tuning.\n\n## Strategic experiment — not applied\n\nPrototype a shared task-plan representation in a proposed packages/service/src/tasks/plan-consistency.ts, retaining explicit prerequisites, alternative branches and assumption provenance before rendering prose and tables. Structural validation can check references/cycles; semantic review must still compare contingency prose with the graph. Keep persistence behind existing task/store boundaries. The current T4/T8/T9 inconsistencies motivate the experiment, not a claim that graph validation alone solves planning.\n\n## Generalization and next experiment\n\nThe earlier core plan also passed at 10.0 in 8m35s; this productivity entry passed in 8m59s. Both are first-round entries in separate suite cells, not completion of the requested three trials per entry. Compact planning is repeatably deliverable in these two observations, while records-intake and incident synthesis stalled. Validate generic dependency review on unrelated plans and report actual consistency alongside the unchanged fixed gate.\n\n## Performance and fit\n\nRecorded decode 2.99 tokens/sec; prefill 161.74. Peak VRAM 10626/12227 MiB, process RSS 9336 MiB, whole-system use 27138/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Launch remains native 0.1.57/CUDA, 65536 context tokens, one slot, f16 KV and 25 CPU FFN layers (39/64 dense FFN layers on GPU). No explicit OOM, context overflow or unexpected native exit found. Shared GPU memory is not separately measured.\n\n## Last three same-scenario trials\n\n- [plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/facts.json): pass, fixed 10, 514.7 seconds, recorded decode 2.88 tokens/sec.\n- [plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h/facts.json): pass, fixed 10, 30.1 seconds, recorded decode 111.68 tokens/sec.\n- [plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-13-849Z-1v9m/facts.json): pass, fixed 10, 30.3 seconds, recorded decode 113.94 tokens/sec.\n\nThe plan comparison includes the earlier Saluki core entry; older external entries use Gemma4 12B/native 0.1.54 and are historical context, not a controlled regression. No current suite entry has completed its requested three repetitions yet.\n" }, { "cellId": "r1-saluki-cuda-productivity-meeting-followup", "baselineCellId": "r1-saluki-cuda-productivity-meeting-followup", "suite": "productivity", "scenario": "meeting-followup", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T07:31:51.779Z", "finishedAt": "2026-11-04T07:45:25.969Z", "durationMs": 814190, "success": true, "reason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 361463 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "meeting-followup", "bytes": 2033, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.21734348647579427 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 361463 }, { "source": "facts", "path": "outcome.durationMs", "value": 814190 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 136, "peakRssMb": 12269, "peakCpuPercent": 1207.8, "firstAtMs": 0, "lastAtMs": 800691 }, "gpu": { "available": true, "sampleCount": 136, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 136, "peakUsedMb": 29895, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 23757, "totalOutputTokens": 1871, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 6312, "todayTokensCached": 0, "todayTokensOut": 1259, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 6312, "totalTokensCached": 0, "totalTokensOut": 1259, "totalCost": 0, "medianOutputTokensPerSec": 3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 3, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3546, "outputTokens": 551, "cost": 0, "durationMs": 342124, "outputTokensPerSec": 2.975420215892163, "at": "2026-11-04T07:43:35.377Z" }, "lastUpdated": "2026-11-04T07:43:35.377Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T07:45:20.669Z" } }, "source": "engine-log" }, "derived": { "durationMs": 802392, "meanTokensPerSec": 2.98, "genTokensPerSec": 2.98, "genTokens": 1871, "promptTokensPerSec": 162.65, "peakRssMb": 12269, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6312, "output": 1259 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T07:31:57.722Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T07:31:57.607Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "noor": 1, "rafiq": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 3, "maxContextFill": 0.0941619873046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0941619873046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "durationMs": 814190, "timeoutMs": 3746098, "budgetUsedFraction": 0.21734348647579427 }, "timing": { "startedAt": "2026-11-04T07:31:51.779Z", "finishedAt": "2026-11-04T07:45:25.969Z", "timeToFirstArtifactMs": 361463, "timeToLastArtifactWriteMs": 807675, "timeToFirstTokenMs": 65094, "firstTurnTtftMs": 53557, "timeToFirstToolCallMs": 105868 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "read_file": 4, "write_file": 3, "list_dir": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 808383, "filePath": "meeting-followup", "bytes": 2033, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } ], "latest": { "filePath": "meeting-followup", "bytes": 2033, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T07:31:51.784Z] [trial] id=meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c scenario=meeting-followup model=underdog-saluki-27b-iq2-mix", "[2026-11-04T07:31:51.784Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "[2026-11-04T07:31:51.784Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-meeting-followup-JiMO17", "[2026-11-04T07:31:51.785Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T07:31:51.785Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T07:31:51.788Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T07:31:51.792Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T07:31:51.800Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-meeting-followup-JiMO17", "[2026-11-04T07:31:51.800Z] [trial] throughput-scaled ceiling: 30m → 62m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T07:31:57.658Z] [trial] daemon spawned pid=30364 port=https://127.0.0.1:52817", "[2026-11-04T07:31:59.131Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T07:31:59.150Z] [trial] meester=malika", "[2026-11-04T07:31:59.151Z] [trial] running scenario setup hook", "[2026-11-04T07:31:59.649Z] [trial] scenario setup complete", "[2026-11-04T07:31:59.650Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T07:31:59.745Z] [poll] initial digests hard=e43244945a15929a soft=e8c5f8b1c224e1b6", "[2026-11-04T07:31:59.827Z] [poll] hard-progress: e43244945a15929a → 9386cceaea89d3ea after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T07:33:40.836Z] [poll] hard-progress: 9386cceaea89d3ea → a14ffeea39f11e34 after 101s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=3,slot=68,stream=86) sniff=0/0B)", "[2026-11-04T07:37:54.576Z] [poll] hard-progress: a14ffeea39f11e34 → d274d5a4284bfae1 after 254s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=709,stream=1358) sniff=1/1726B)", "[2026-11-04T07:39:17.728Z] [poll] hard-progress: d274d5a4284bfae1 → 4be0b29a1301b47e after 83s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=854,stream=1502) sniff=1/1726B)", "[2026-11-04T07:40:11.231Z] [poll] hard-progress: 4be0b29a1301b47e → 715901170cfcaf9e after 54s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=952,stream=1613) sniff=1/1726B)", "[2026-11-04T07:43:38.575Z] [poll] hard-progress: 715901170cfcaf9e → cf37fe755f0a3ec8 after 207s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=3,tools=8,slot=1470,stream=2637) sniff=1/1726B)", "[2026-11-04T07:45:20.162Z] [poll] terminal: success=true reason=meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "[2026-11-04T07:45:20.708Z] [capture] dumped 2 session(s)", "[2026-11-04T07:45:20.711Z] [capture] snapshotted telemetry for 2 session(s)" ] }, "metrics": { "process": { "sampleCount": 136, "peakRssMb": 12269, "peakCpuPercent": 1207.8, "firstAtMs": 0, "lastAtMs": 800691 }, "gpu": { "available": true, "sampleCount": 136, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 136, "peakUsedMb": 29895, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 23757, "totalOutputTokens": 1871, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 6312, "todayTokensCached": 0, "todayTokensOut": 1259, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 6312, "totalTokensCached": 0, "totalTokensOut": 1259, "totalCost": 0, "medianOutputTokensPerSec": 3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 3, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3546, "outputTokens": 551, "cost": 0, "durationMs": 342124, "outputTokensPerSec": 2.975420215892163, "at": "2026-11-04T07:43:35.377Z" }, "lastUpdated": "2026-11-04T07:43:35.377Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T07:45:20.669Z" } }, "source": "engine-log" }, "derived": { "durationMs": 802392, "meanTokensPerSec": 2.98, "genTokensPerSec": 2.98, "genTokens": 1871, "promptTokensPerSec": 162.65, "peakRssMb": 12269, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6312, "output": 1259 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T07:31:51.779Z", "finishedAt": "2026-11-04T07:45:25.969Z", "timeToFirstArtifactMs": 361463, "timeToLastArtifactWriteMs": 807675, "timeToFirstTokenMs": 65094, "firstTurnTtftMs": 53557, "timeToFirstToolCallMs": 105868 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T07:31:57.722Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T07:31:57.607Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "noor": 1, "rafiq": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 3, "maxContextFill": 0.0941619873046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0941619873046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-meeting-followup", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "scenarioId": "meeting-followup", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/workspace/orchid-launch-follow-up/meeting-brief.md", "axisSource": "Declared judge.axes in evals/src/scenarios/meeting-followup.ts", "artifactKind": "markdown", "artifactBytes": 1732, "supportingArtifactPaths": [ "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/workspace/orchid-launch-follow-up/action-items.csv" ], "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ], "scoreAxes": { "decisionClarity": 7, "grounding": 8, "actionability": 8, "brevity": 7 }, "meanScore": 7.5, "justification": "Clear decisions and open questions, accurate owners/dates, and an executable four-row register with the dependency preserved. Legal-review and unassigned rollback risks remain explicit. The stale agenda is repeatedly discussed, and scheduling the review appears under both decisions and actions, adding minor repetition.", "reviewBasis": "Full declared Markdown artifact below 20 KiB, companion CSV, authoritative transcript, roster and stale agenda read. Model identity was visible in paths." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:33.909Z", "reviewId": "015", "cellId": "r1-saluki-cuda-productivity-meeting-followup", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "scenarioId": "meeting-followup", "status": "scored", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "baselineCellId": "r1-saluki-cuda-productivity-meeting-followup", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "773270cf79d480481efc484178e3bb25bcf202a95735320a55a49c11e4022446", "evidenceSha256": "bf77ec9d7d293d941f7ebd8ea0e8e4081e3e47b73d7ccb365265883818f0f385", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/workspace/orchid-launch-follow-up/meeting-brief.md", "bytes": 1732, "sha256": "1523b6e14ee577164ab0371a70a01f5e296b2c9d06641fa2329c0e3aa51276e8", "shownBytes": 1732, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c\\workspace\\orchid-launch-follow-up\\meeting\\old-agenda.md", "bytes": 262, "sha256": "8a895418b408627cf5fd7367a45341bedc5fc29515c61dac05de3b47f6fdde38", "shownBytes": 262, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c\\workspace\\orchid-launch-follow-up\\meeting\\roster.md", "bytes": 223, "sha256": "8f77db20d7700521aecdf6cddcbbe807ebf99bae5c2043a558fd5f96831cad69", "shownBytes": 223, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c\\workspace\\orchid-launch-follow-up\\meeting\\transcript.md", "bytes": 977, "sha256": "7a4039bda8b643bfdfb3f8d1208a5c7835fd3b13f280867db9b6915992e0800a", "shownBytes": 977, "truncated": false } ], "scoreAxes": { "decisionClarity": 8, "grounding": 8, "actionability": 8, "brevity": 8 }, "justification": "The brief has the required H2 order. It separates the three real decisions from the stale proposals and keeps the annual plan and rollback owner open. Owners, dates and the A1→A2 dependency match the transcript and roster. Minor flaw: \"Jordan will schedule a review\" is listed under Decisions but is really an action. It is compact.", "limitations": "action-items.csv was not shown, so its header, row count, empty depends_on cells and status values could not be checked. Scores reflect the brief only. The brief's table uses \"—\" for empty dependencies, which is not a CSV check.", "meanScore": 8, "durationMs": 15614, "costUsd": 0.023018, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4411, "cache_read_input_tokens": 0, "output_tokens": 537, "output_tokens_details": { "thinking_tokens": 204 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4411, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 537, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 4411, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4411 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 537, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 4411, "webSearchRequests": 0, "costUSD": 0.023018, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 204, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4411, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - meeting-followup / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 13m 34s; budget used: 21.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 361463`; `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2033,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.21734348647579427`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 361463`; `facts.outcome.durationMs: 814190` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12269 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29895 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.98 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 23757 + 1871 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 9% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 361463`\n- `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2033,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.21734348647579427`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 361463`\n- `facts.outcome.durationMs: 814190`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 361463`; `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2033,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.21734348647579427`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 361463`; `facts.outcome.durationMs: 814190`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nDeclared axes from evals/src/scenarios/meeting-followup.ts, scored by codex-inline/current-session after reading the complete 1732-byte brief, its 307-byte CSV companion and all supplied meeting sources:\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| decisionClarity | 7 | Separates decisions, open questions, actions, and risks cleanly. |\n| grounding | 8 | Owners, dates, scope, and dependencies match the transcript and roster. |\n| actionability | 8 | A reader can execute the follow-up without rereading the transcript. |\n| brevity | 7 | The brief is scannable and avoids transcript-style repetition. |\n\n**Advisory mean: 7.5/10. Fixed composite remains 10.0.**\n\nJustification: Clear decisions and open questions, accurate owners/dates, and an executable four-row register with the dependency preserved. Legal-review and unassigned rollback risks remain explicit. The stale agenda is repeatedly discussed, and scheduling the review appears under both decisions and actions, adding minor repetition.\n\n## Positives\n\n- All 13 reconciliation gates passed: the confirmed launch/scope/migration decisions, unresolved annual-plan and rollback questions, legal risk and all four exact action rows (sniff.latest.score: 13/13; outcome.success: true).\n- Both human-readable and machine-readable outputs were delivered, with no user intervention, missing roles or fixed-rubric red flags (artifacts.otherFileCount: 6; toolUse.byTool.write_file: 3; autoAnswer.total: 0; team.missingExpectedRoles: []; toolUse.redFlags: []). The CSV and Markdown table agree.\n\n## Negatives\n\n- The model wrote the brief, inspected files again, rewrote the brief, then wrote the CSV (toolUse.byTool.write_file: 3; read_file: 4; list_dir: 2; recording/chat-events.jsonl). First output was at 361463 ms; all outputs closed at 807675 ms of an 814190 ms trial (timing; outcome.durationMs).\n- The fixed score does not measure the minor repetition identified by the declared brevity/decisionClarity axes. The advisory review above is separate and must not be merged into capability scores.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/pl/planner/versions/1.1.0/about.md, add: “For paired narrative and structured outputs, extract the action list once, use it consistently in both, and separate an agreed action to hold a review from the unresolved decision it will address.” Validate on another meeting domain and a project-handoff task, measuring agreement, redundancy and duplicate writes without adding scenario-specific field values.\n\n## Strategic experiment — not applied\n\nPrototype a reusable validated action-register intermediate in a proposed packages/service/src/tasks/action-register.ts, containing owner, due date, prerequisite and decision status, then render both CSV and narrative via existing write boundaries. This targets cross-artifact consistency and avoids regenerating the same facts independently. This trial's matching outputs are a positive baseline; the third write and rereads motivate an efficiency experiment rather than a correctness fix.\n\n## Generalization and next experiment\n\nThis task successfully combined source precedence, unresolved questions and exact CSV output, contrasting with conflict-synthesis's 31-call repair sequence and records-intake's missing CSV. The useful next experiment is single-extraction paired-output authoring on two unrelated follow-up tasks, measuring fixed acceptance, advisory clarity and time. Do not infer that all structured-data tasks are reliable from this one pass.\n\n## Performance and fit\n\nRecorded decode 2.98 tokens/sec; prefill 162.65. Peak VRAM 10626/12227 MiB, process RSS 12269 MiB, whole-system use 29895/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Launch remains native 0.1.57/CUDA, 65536 context tokens, one slot, f16 KV and 25 CPU FFN layers (39/64 dense FFN layers on GPU). No explicit OOM, context overflow or unexpected native exit found. Shared GPU memory is not separately measured.\n\n## Last three same-scenario trials\n\n- [meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/facts.json): pass, fixed 10, 169.6 seconds, recorded decode 130.36 tokens/sec.\n- [meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9/facts.json): pass, fixed 10, 146.2 seconds, recorded decode 139.08 tokens/sec.\n- [meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-21-19-336Z-krho/facts.json): pass, fixed 10, 172.3 seconds, recorded decode 130.68 tokens/sec.\n\nThe plan comparison includes the earlier Saluki core entry; older external entries use Gemma4 12B/native 0.1.54 and are historical context, not a controlled regression. No current suite entry has completed its requested three repetitions yet.\n" }, { "cellId": "r1-saluki-cuda-productivity-craftbook-spreadsheet-model", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-spreadsheet-model", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T07:47:54.586Z", "finishedAt": "2026-11-04T09:05:39.074Z", "durationMs": 4664488, "success": false, "reason": "hit hard ceiling (3746098ms + 15m progress extensions) — last hard progress 1200s before the end; the deliverable never closed", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "failureMode": "timeout", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-spreadsheet-model", "score": 3, "bytes": 3059, "failReason": "index.html is 0 bytes, need ≥ 800", "deliverableMissing": true, "milestones": 2 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "timeout" }, { "source": "facts", "path": "outcome.reason", "value": "hit hard ceiling (3746098ms + 15m progress extensions) — last hard progress 1200s before the end; the deliverable never closed" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1798028 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-spreadsheet-model", "bytes": 3059, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook spreadsheet-model has not finished: its gates have not accepted the result yet" } } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1798028 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-spreadsheet-model", "bytes": 3059, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook spreadsheet-model has not finished: its gates have not accepted the result yet" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1798028 }, { "source": "facts", "path": "outcome.durationMs", "value": 4664488 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.5, "band": "framework-gap" }, "facts": { "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 786, "peakRssMb": 11429, "peakCpuPercent": 1209.9, "firstAtMs": 0, "lastAtMs": 4650262 }, "gpu": { "available": true, "sampleCount": 786, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 786, "peakUsedMb": 28896, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 19743, "totalOutputTokens": 9002, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 9043, "todayTokensCached": 7875, "todayTokensOut": 4467, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 9043, "totalTokensCached": 7875, "totalTokensOut": 4467, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9043, "cachedInputTokens": 7875, "outputTokens": 4467, "cost": 0, "durationMs": 1781327, "outputTokensPerSec": 2.7351047609428414, "at": "2026-11-04T08:17:52.613Z" }, "lastUpdated": "2026-11-04T08:17:52.613Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T09:05:32.925Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4651825, "meanTokensPerSec": 2.73, "genTokensPerSec": 2.73, "genTokens": 9002, "promptTokensPerSec": 164.36, "peakRssMb": 11429, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 9043, "output": 4467 } } } }, "craftbook": { "craftbookId": "spreadsheet-model", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T07:48:02.604Z", "completedAt": "2026-11-04T08:17:52.784Z", "ms": 1790180, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "b5f6a62c-0692-4038-bc36-b2d0c2b42885" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "build", "activatedAt": "2026-11-04T08:17:52.785Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "e910716a-3620-45eb-9c25-4dd8f4dd7658" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T07:48:00.804Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T07:48:00.690Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1790180 }, "sessions": { "total": 2, "taskScoped": 2, "byGezel": { "henrik": 1, "sione": 1 }, "perTask": { "spreadsheet-financial-model-workflow-eval/1": 2 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.5, "maxMessages": 2, "maxContextFill": 0.1226348876953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1226348876953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 3, "citations": [ "knowledge://bendyline/handboek/model-scorecard", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/ops-runbook" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "timeout", "reason": "hit hard ceiling (3746098ms + 15m progress extensions) — last hard progress 1200s before the end; the deliverable never closed", "durationMs": 4664488, "timeoutMs": 3746098, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-04T07:47:54.586Z", "finishedAt": "2026-11-04T09:05:39.074Z", "timeToFirstArtifactMs": 1798028, "timeToLastArtifactWriteMs": 1798028, "timeToFirstTokenMs": 73341, "firstTurnTtftMs": 56641, "timeToFirstToolCallMs": 155391 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Planner", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 8, "byTool": { "read_files": 1, "write_artifact": 1, "read_artifact": 1, "read_file": 3, "list_dir": 1, "list_artifacts": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 8295, "filePath": "craftbook-spreadsheet-model", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, source/brief.md, source/records.csv" }, { "atMs": 157761, "filePath": "craftbook-spreadsheet-model", "bytes": 0, "score": 2, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook spreadsheet-model has not finished: its gates have not accepted the result yet" }, { "atMs": 1802029, "filePath": "craftbook-spreadsheet-model", "bytes": 3059, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook spreadsheet-model has not finished: its gates have not accepted the result yet" } ], "latest": { "filePath": "craftbook-spreadsheet-model", "bytes": 3059, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook spreadsheet-model has not finished: its gates have not accepted the result yet" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T07:47:54.591Z] [trial] id=craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz scenario=craftbook-spreadsheet-model model=underdog-saluki-27b-iq2-mix", "[2026-11-04T07:47:54.591Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "[2026-11-04T07:47:54.591Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T07:47:54.591Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-spreadsheet-model-ImyqEt", "[2026-11-04T07:47:54.591Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T07:47:54.595Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T07:47:54.598Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T07:47:54.605Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-spreadsheet-model-ImyqEt", "[2026-11-04T07:47:54.605Z] [trial] throughput-scaled ceiling: 30m → 62m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T07:48:00.784Z] [trial] daemon spawned pid=40204 port=https://127.0.0.1:57490", "[2026-11-04T07:48:02.067Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T07:48:02.085Z] [trial] meester=torsten", "[2026-11-04T07:48:02.085Z] [trial] running scenario setup hook", "[2026-11-04T07:48:02.775Z] [trial] scenario setup complete", "[2026-11-04T07:48:02.778Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T07:48:02.833Z] [poll] initial digests hard=b81d4291f71594e4 soft=ecbffb0a46d1d0e1", "[2026-11-04T07:48:02.926Z] [poll] hard-progress: b81d4291f71594e4 → b8fcbb9d5ebf92aa after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T07:48:09.699Z] [poll] hard-progress: b8fcbb9d5ebf92aa → bee50cc9799fa4a2 after 7s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T07:50:32.397Z] [poll] hard-progress: bee50cc9799fa4a2 → 8e3f86fcd6f04481 after 143s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=215,stream=243) sniff=2/0B)", "[2026-11-04T08:17:56.662Z] [poll] hard-progress: 8e3f86fcd6f04481 → c994c90e75aae1a8 after 1644s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=4581,stream=5675) sniff=3/3059B)", "[2026-11-04T08:19:43.617Z] [poll] hard-progress: c994c90e75aae1a8 → 8e82d1d14425f1d5 after 107s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=4649,stream=5775) sniff=3/3059B)", "[2026-11-04T08:45:32.536Z] [poll] hard-progress: 8e82d1d14425f1d5 → 0c0dac7800b0dc03 after 1549s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=8734,stream=9870) sniff=3/3059B)", "[2026-11-04T08:50:31.448Z] [poll] hard ceiling reached but hard progress moved 299s ago — extending 15m (77m of 125m cap)", "[2026-11-04T09:05:32.384Z] [poll] hard ceiling reached after 3746098ms", "[2026-11-04T09:05:33.043Z] [capture] dumped 2 session(s)" ] }, "metrics": { "process": { "sampleCount": 786, "peakRssMb": 11429, "peakCpuPercent": 1209.9, "firstAtMs": 0, "lastAtMs": 4650262 }, "gpu": { "available": true, "sampleCount": 786, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 786, "peakUsedMb": 28896, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 19743, "totalOutputTokens": 9002, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 9043, "todayTokensCached": 7875, "todayTokensOut": 4467, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 9043, "totalTokensCached": 7875, "totalTokensOut": 4467, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9043, "cachedInputTokens": 7875, "outputTokens": 4467, "cost": 0, "durationMs": 1781327, "outputTokensPerSec": 2.7351047609428414, "at": "2026-11-04T08:17:52.613Z" }, "lastUpdated": "2026-11-04T08:17:52.613Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T09:05:32.925Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4651825, "meanTokensPerSec": 2.73, "genTokensPerSec": 2.73, "genTokens": 9002, "promptTokensPerSec": 164.36, "peakRssMb": 11429, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 9043, "output": 4467 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T07:47:54.586Z", "finishedAt": "2026-11-04T09:05:39.074Z", "timeToFirstArtifactMs": 1798028, "timeToLastArtifactWriteMs": 1798028, "timeToFirstTokenMs": 73341, "firstTurnTtftMs": 56641, "timeToFirstToolCallMs": 155391 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T07:48:02.604Z", "completedAt": "2026-11-04T08:17:52.784Z", "ms": 1790180, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "b5f6a62c-0692-4038-bc36-b2d0c2b42885" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "build", "activatedAt": "2026-11-04T08:17:52.785Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "e910716a-3620-45eb-9c25-4dd8f4dd7658" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T07:48:00.804Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T07:48:00.690Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1790180 }, "sessions": { "total": 2, "taskScoped": 2, "byGezel": { "henrik": 1, "sione": 1 }, "perTask": { "spreadsheet-financial-model-workflow-eval/1": 2 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.5, "maxMessages": 2, "maxContextFill": 0.1226348876953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1226348876953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "scenarioId": "craftbook-spreadsheet-model", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "artifactKind": "markdown", "artifactBytes": 3120, "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/sp/spreadsheet-model/versions/1.0.5/test.json rubric mapped to judge.axes by evals/src/craftbooks/scenario.ts", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": { "workflow fidelity": 5, "usefulness": 3 }, "meanScore": 4, "justification": "The scope follows the required first phase, labels assumptions, and specifies formulas and checks. Its default cumulative value is wrong (18,036 versus 8,436), and payback treats a recurring monthly cost as an upfront amount. The downstream calculator and completed validation are absent.", "reviewBasis": "Complete primary scope artifact and all three source fixtures reviewed. Native rubric targets scope.md, not index.html. Model identity visible in paths. Arithmetic checked independently; no artifacts repaired.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/workspace/spreadsheet-financial-model-workflow-eval/index.html", "status": "noArtifact", "reason": "Required final calculator was not committed before the hard ceiling; no qualitative scores invented." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:38.118Z", "reviewId": "016", "cellId": "r1-saluki-cuda-productivity-craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "scenarioId": "craftbook-spreadsheet-model", "status": "scored", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-spreadsheet-model", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "4f175b3a82694e2fb1a37cf64285501846d02539264a371c147ed6d64ece6fb9", "evidenceSha256": "f3f4d6a8b895419f50b6a1fe4cb91ff726eb18c0d1adf02bcaeaaee757d31fce", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "bytes": 3120, "sha256": "757cab1e8f9a997d02a06feb526df76cc8c2660e44c7adb4aef1d6650536bca5", "shownBytes": 3120, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\brief.md", "bytes": 791, "sha256": "9b7893b96d2e07d87b152dcdef1c8e16395615b8bc827a34cc10dd0706ab0bda", "shownBytes": 791, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\craftbook-eval-brief.md", "bytes": 2268, "sha256": "6cebbc3d1462d96e4ee669842d5d1d5fcc2c9e0c95bf6e723aefa688b297cbdd", "shownBytes": 2268, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\records.csv", "bytes": 141, "sha256": "430befe636c28a4b2751e842a79773440af0bd7a5f1b742ea177c6ef204d74c6", "shownBytes": 141, "truncated": false } ], "scoreAxes": { "workflow fidelity": 6, "usefulness": 5 }, "justification": "scope.md covers the scope-step content: question, output, A1–A6 drivers, C1–C5 formulas, time axis, two sanity scenarios and an acceptance checklist. But S1's expected value is wrong. 2,703×12 − 24,000 = 8,436, not 18,036. S2 has a small slip (−21,426.72 vs −21,426.74). The contract fails its own hand-check.", "limitations": "Only the scope artifact was shown. The missing index.html is expected at the scope step, so it was not penalised. Handoff via advance_task_step and gate results were not shown. No rendering or execution was done.", "meanScore": 5.5, "durationMs": 17057, "costUsd": 0.03338, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6419, "cache_read_input_tokens": 0, "output_tokens": 770, "output_tokens_details": { "thinking_tokens": 430 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6419, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 770, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 6419, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6419 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 770, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 6419, "webSearchRequests": 0, "costUSD": 0.03338, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 430, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6419, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-spreadsheet-model / underdog-saluki-27b-iq2-mix\n\n**Composite: 4.5 / 10** (band: framework-gap)\n**Outcome: timeout** (duration: 77m 44s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (3746098ms + 15m progress extensions) — last hard progress 1200s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 1798028`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3059,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook spreadsheet-model has not finished: its gate...` |\n| Output quality (25%) | 5 / 10 | The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1798028`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3059,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook spreadsheet-model has not finished: its gate...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 1798028`; `facts.outcome.durationMs: 4664488` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 5 + 0.2 * 2.5 + 0.15 * 10 = **4.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11429 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10636 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28896 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.73 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 19743 + 9002 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 4 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1790s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 2 / 0.50 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 12% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 3 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"timeout\"`\n- `facts.outcome.reason: \"hit hard ceiling (3746098ms + 15m progress extensions) — last hard progress 1200s before the end; the deliverable never closed\"`\n- `facts.timing.timeToFirstArtifactMs: 1798028`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3059,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook spreadsheet-model has not finished: its gate...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 50% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 1798028`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3059,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook spreadsheet-model has not finished: its gate...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 8`\n- `facts.timing.timeToFirstArtifactMs: 1798028`\n- `facts.outcome.durationMs: 4664488`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (3746098ms + 15m progress extensions) — last hard progress 1200s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 1798028`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3059,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook spreadsheet-model has not finished: its gate...`\n- Quality: The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1798028`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3059,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook spreadsheet-model has not finished: its gate...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 1798028`; `facts.outcome.durationMs: 4664488`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nThe declared primary judge artifact is the 3120-byte scope.md, which exists. The current Gilde test.json rubric is mapped directly to scenario judge.axes by evals/src/craftbooks/scenario.ts. Read the whole scope and the three seeded sources; score the scope even though the final HTML is missing and the workflow failed. Record the separate index.html deliverable as noArtifact, without scores. Fixed composite remains 4.5.\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| workflow fidelity | 5 | The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute. |\n| usefulness | 3 | The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief. |\n\n**Advisory mean: 4.0/10, codex-inline/current-session.**\n\nThe scope follows the required first phase, labels assumptions, and specifies formulas and checks. Its default cumulative value is wrong (18,036 versus 8,436), and payback treats a recurring monthly cost as an upfront amount. The downstream calculator and completed validation are absent.\n\nThe scope's S1 subtraction should be 32,436 - 24,000 = 8,436. Its recurring program cost has units USD/month; dividing that by monthly recovery does not establish a payback duration. The S2 negative-net example says “never pays back,” while the stated C5 rule yields a positive ratio whenever recovery is positive. These are advisory correctness findings in an intermediate document, not edits to fixed gates. The S2 value -21,426.74 is consistent with rounding after full-precision calculation.\n\n## Positives\n\n- The requested craftbook was invoked and the scope phase passed its runtime checkpoint, handing off to the Developer (craftbook.craftbookTasks: 1; continuity.steps.completed: 1; gateApprovals: 1). The planner labeled added revenue, recovery and cost assumptions separately from the supplied leakage rates.\n- The trial had no missing expected role, tool red flag or user intervention (team.missingExpectedRoles: []; toolUse.redFlags: []; autoAnswer.total: 0). Native loading and ordinary source/artifact access worked.\n\n## Negatives\n\n- The workflow stopped at build, with no calculator and no terminal step (artifacts.htmlFiles: []; craftbook.reachedTerminal: false; sniff.latest.score: 3/6; continuity.steps.perStep). Eight completed calls included one scope write but no completed workspace write (toolUse.byTool).\n- First output took 1,798,028 ms, about 30 minutes. The entire attempt took 4,664,488 ms, 77m44s, including the standard 15-minute progress extension (timing.timeToFirstArtifactMs; outcome.durationMs; outcome.reason). The scope math defects show that an accepted intermediate gate did not establish numerical correctness.\n- Build reasoning continued until 08:50:42Z; write_file arguments streamed from 08:51:15Z to 09:05:32Z but never completed. Its incomplete argument stream was 6683 characters (recording/chat-events.jsonl; advisory-generation-evidence.json; outcome.failureMode: timeout). The fixed 9002 generated-token total covers completed requests; an unfinished request separately reached 3252 request-local tokens. Do not report 9002 as all generation or treat streamed arguments as a saved file.\n\n## Tactical experiment — not applied\n\nFor a later isolated A/B, reduce tuning.reasoning.thinkingBudget and both thinking-general/thinking-coding profile overrides from 4096 to 1024 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json. Preserve output caps, sampling, context and deadlines. At the observed 2.73 tokens/sec, a 4096-token reasoning allowance can consume about 25 minutes per provider iteration. Measure earlier completed writes and correctness on a model/calculator task, incident synthesis and an unrelated coding task; reject if output correctness degrades. This campaign remains unchanged.\n\n## Strategic experiment — not applied\n\nPrototype a reusable, deterministic worked-example checker in a proposed packages/script-stdlib/src/validate-model-examples.ts, exposed through the existing installed-script boundary. Given declared inputs, expressions, units and expected outputs, calculate examples and report mismatches before a downstream builder relies on them. The S1 error and monthly-cost/payback mismatch motivate testing arithmetic and unit checks; they do not justify fixture-specific expected values or silently repairing scored artifacts. Separately investigate first-deliverable reasoning allocation in packages/core/src/local-loop/constrained-turn.ts so slow decoding leaves time for the atomic write to finish.\n\n## Generalization and next experiment\n\nIncident-postmortem also timed out with an unfinished first write; records-intake spent long intervals reasoning and rereading without a CSV. Meeting-followup successfully produced a brief and exact CSV at about 2.98 tokens/sec. This supports experiments on time to complete a write across task shapes. It does not show a load failure or establish a universal task ceiling. The separate budget A/B should measure completed artifacts, fixed acceptance and advisory correctness across all three task families; implement none of it during this frozen campaign.\n\n## Performance and fit\n\nRecorded decode 2.73 tokens/sec; prefill 164.36. Peak VRAM 10636/12227 MiB, process RSS 11429 MiB, whole-system RAM 28896/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57/CUDA retained 65536 context tokens, one slot, f16 KV and 25 CPU FFN layers: dense FFN of 39/64 layers on GPU. This is mixed CPU/GPU execution, not a fully GPU-resident model. No explicit OOM, context overflow or unexpected native exit was found. Recorded context fill was about 12.3%, with zero compactions (continuity.sessions.maxContextFill; continuity.compaction). Shared GPU memory is not separately measured.\n\n## Last three same-scenario trials\n\n- [craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/facts.json): pass, fixed 9.5, 147.4 seconds, recorded decode 128.8 tokens/sec.\n- [craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9/facts.json): pass, fixed 9.5, 147.4 seconds, recorded decode 125.02 tokens/sec.\n- [craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-29-33-156Z-p9zc/facts.json): pass, fixed 9.5, 159.3 seconds, recorded decode 124.76 tokens/sec.\n\nThese older Gemma4 12B/native 0.1.54 results are historical context, not a controlled regression. All three completed their six deterministic checks; the current trial stopped at 3/6. Different model and runtime identities prevent attributing the difference to native 0.1.57.\n" }, { "cellId": "r1-saluki-cuda-productivity-conflict-synthesis", "baselineCellId": "r1-saluki-cuda-productivity-conflict-synthesis", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T09:08:33.935Z", "finishedAt": "2026-11-04T09:37:29.216Z", "durationMs": 1735281, "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1427767 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 2767, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.4632235995961665 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1427767 }, { "source": "facts", "path": "outcome.durationMs", "value": 1735281 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 292, "peakRssMb": 9872, "peakCpuPercent": 1218.8, "firstAtMs": 1, "lastAtMs": 1721422 }, "gpu": { "available": true, "sampleCount": 292, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 292, "peakUsedMb": 27913, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12755, "totalOutputTokens": 4377, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12743, "todayTokensCached": 11796, "todayTokensOut": 705, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12743, "totalTokensCached": 11796, "totalTokensOut": 705, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 12743, "cachedInputTokens": 11796, "outputTokens": 705, "cost": 0, "durationMs": 1715215, "outputTokensPerSec": 2.7094166221139537, "at": "2026-11-04T09:37:20.668Z" }, "lastUpdated": "2026-11-04T09:37:20.668Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T09:37:23.253Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1722852, "meanTokensPerSec": 2.7, "genTokensPerSec": 2.7, "genTokens": 4377, "promptTokensPerSec": 158.58, "peakRssMb": 9872, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 12743, "output": 705 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T09:08:39.919Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T09:08:39.801Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.142425537109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.142425537109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 1735281, "timeoutMs": 3746098, "budgetUsedFraction": 0.4632235995961665 }, "timing": { "startedAt": "2026-11-04T09:08:33.935Z", "finishedAt": "2026-11-04T09:37:29.216Z", "timeToFirstArtifactMs": 1427767, "timeToLastArtifactWriteMs": 1726734, "timeToFirstTokenMs": 86698, "firstTurnTtftMs": 75180, "timeToFirstToolCallMs": 153639 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 8, "byTool": { "read_file": 7, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 1432909, "filePath": "conflict-synthesis", "bytes": 2767, "score": 12, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "citations" ], "failReason": null }, { "atMs": 1728779, "filePath": "conflict-synthesis", "bytes": 2767, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } ], "latest": { "filePath": "conflict-synthesis", "bytes": 2767, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T09:08:33.939Z] [trial] id=conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew scenario=conflict-synthesis model=underdog-saluki-27b-iq2-mix", "[2026-11-04T09:08:33.940Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "[2026-11-04T09:08:33.940Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-IndoMO", "[2026-11-04T09:08:33.940Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T09:08:33.940Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T09:08:33.944Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T09:08:33.948Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T09:08:33.955Z] [trial] throughput-scaled ceiling: 30m → 62m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T09:08:33.955Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-IndoMO", "[2026-11-04T09:08:39.903Z] [trial] daemon spawned pid=42368 port=https://127.0.0.1:58580", "[2026-11-04T09:08:41.329Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T09:08:41.347Z] [trial] meester=priya", "[2026-11-04T09:08:41.347Z] [trial] running scenario setup hook", "[2026-11-04T09:08:41.823Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T09:08:41.822Z] [trial] scenario setup complete", "[2026-11-04T09:08:41.906Z] [poll] initial digests hard=b5feb54518cdde22 soft=cb6d8b4c9c8ce0ae", "[2026-11-04T09:08:42.030Z] [poll] hard-progress: b5feb54518cdde22 → e838a485f34e5fa1 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T09:11:10.636Z] [poll] hard-progress: e838a485f34e5fa1 → fe0b038391450f9b after 149s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=75,stream=106) sniff=0/0B)", "[2026-11-04T09:32:26.897Z] [poll] hard-progress: fe0b038391450f9b → 3e46b93dec09ea45 after 1276s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=3436,stream=4259) sniff=12/2767B)", "[2026-11-04T09:32:56.352Z] [poll] hard-progress: 3e46b93dec09ea45 → 093554645cdfc6c8 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=3480,stream=4309) sniff=12/2767B)", "[2026-11-04T09:37:22.714Z] [poll] terminal: success=true reason=all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "[2026-11-04T09:37:23.327Z] [capture] dumped 1 session(s)", "[2026-11-04T09:37:23.329Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T09:37:23.382Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T09:37:29.215Z] [trial] daemon shut down" ] }, "metrics": { "process": { "sampleCount": 292, "peakRssMb": 9872, "peakCpuPercent": 1218.8, "firstAtMs": 1, "lastAtMs": 1721422 }, "gpu": { "available": true, "sampleCount": 292, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 292, "peakUsedMb": 27913, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12755, "totalOutputTokens": 4377, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12743, "todayTokensCached": 11796, "todayTokensOut": 705, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12743, "totalTokensCached": 11796, "totalTokensOut": 705, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 12743, "cachedInputTokens": 11796, "outputTokens": 705, "cost": 0, "durationMs": 1715215, "outputTokensPerSec": 2.7094166221139537, "at": "2026-11-04T09:37:20.668Z" }, "lastUpdated": "2026-11-04T09:37:20.668Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T09:37:23.253Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1722852, "meanTokensPerSec": 2.7, "genTokensPerSec": 2.7, "genTokens": 4377, "promptTokensPerSec": 158.58, "peakRssMb": 9872, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 12743, "output": 705 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T09:08:33.935Z", "finishedAt": "2026-11-04T09:37:29.216Z", "timeToFirstArtifactMs": 1427767, "timeToLastArtifactWriteMs": 1726734, "timeToFirstTokenMs": 86698, "firstTurnTtftMs": 75180, "timeToFirstToolCallMs": 153639 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T09:08:39.919Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T09:08:39.801Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.142425537109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.142425537109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "scenarioId": "conflict-synthesis", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/workspace/skylark-launch-synthesis/synthesis.md", "artifactBytes": 2779, "reason": "No declared judge axes; full synthesis and six source files reviewed in postmortem without invented scores." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:44.312Z", "reviewId": "017", "cellId": "r1-saluki-cuda-productivity-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "scenarioId": "conflict-synthesis", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew", "baselineCellId": "r1-saluki-cuda-productivity-conflict-synthesis", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "e989b45651d5dbfd6da9490af2e7d60b316b5d08213d3ede0b52ec8553312d97", "evidenceSha256": "66d196587b567bd25313298eb5792d8b489f16efd3b539bcf84c777baac532ab", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/workspace/skylark-launch-synthesis/synthesis.md", "bytes": 2779, "sha256": "53f47ac9403de30d733875e79d1ed1a1042e87e7e5e592abbf9d4465036465d2", "shownBytes": 2779, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. Qualitatively, synthesis.md has the four required sections in order, with winning values (2026-09-01, 210,000 EUR, Marcus) used outside the conflicts section. Each conflict quotes both values and files and states the winner. It cites backticked files well over six times.", "limitations": "Source files were not shown, so quoted values and the conflict set cannot be checked against them. The Priya/oldplan DRI mention appears only in the conflicts section. Contractor-count conflict is somewhat inferred. No runtime verification.", "meanScore": null, "durationMs": 14155, "costUsd": 0.014122000000000001, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2762, "cache_read_input_tokens": 700, "output_tokens": 300, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2762, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 300, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2762, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2762 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 300, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2762, "webSearchRequests": 0, "costUSD": 0.014122000000000001, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2762, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - conflict-synthesis / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 28m 55s; budget used: 46.3%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1427767`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2767,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.4632235995961665`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 1427767`; `facts.outcome.durationMs: 1735281` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9872 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27913 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.7 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 12755 + 4377 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 14% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1427767`\n- `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2767,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.4632235995961665`\n- `facts.toolUse.totalToolCalls: 8`\n- `facts.timing.timeToFirstArtifactMs: 1427767`\n- `facts.outcome.durationMs: 1735281`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1427767`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2767,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.4632235995961665`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 1427767`; `facts.outcome.durationMs: 1735281`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nRead the complete synthesis and all six source files. No judge axes are declared; record notApplicable without invented scores. The final report uses the authoritative September 1 launch, EUR 210,000 budget and Marcus ownership, isolates superseded claims in its conflict section, and cites source filenames. Fixed composite remains 10.0.\n\nOne advisory issue remains: the overview claims the sources disagree on contractor count. The product memo says two contractors; the finance file gives their cost without a count. Silence is not a contradictory count. The conflict section then declares EUR 40,000 the “winner” for a headcount question, conflating cost with quantity, though the open question appropriately leaves headcount unresolved. The introductory claim that each conflict is resolved also overstates that unresolved item.\n\n## Positives\n\n- All 13 reconciliation and provenance checks passed with one completed write and seven reads (sniff.latest.score: 13/13; toolUse.byTool). Source precedence, ownership, budget and retained feature scope are grounded in the supplied documents.\n- No fixed-rubric red flag, missing role or intervention occurred (toolUse.redFlags: []; team.missingExpectedRoles: []; autoAnswer.total: 0). Eight calls compare favorably with 31 in the earlier core entry of the same scenario.\n\n## Negatives\n\n- First delivery took 1,427,767 ms, about 23m48s, with total elapsed 1,735,281 ms, 28m55s (timing.timeToFirstArtifactMs; outcome.durationMs). A fixed efficiency 10.0 reflects the relative budget and call count, not interactive speed.\n- The first poll saw 12/13, lacking committed source-read evidence; final success arrived after the turn committed (sniff.progression; log.txt). The recorded sequence is six source reads, one write, then a readback; there was no document rewrite to obtain the final signal (toolUse.byTool; recording/chat-events.jsonl). The byte-stable report still has the count-versus-cost inconsistency described above.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/re/researcher/versions/1.2.0/about.md, add: “Distinguish contradictory claims from missing information. Resolve values only when they answer the same question and use the same units; leave unsupported quantities open.” Test on an unrelated policy comparison and a budget reconciliation, measuring invented conflicts without changing scenario gates. This targets a reusable evidence distinction, not exact fixture answers.\n\n## Strategic experiment — not applied\n\nPrototype typed claim reconciliation in a proposed packages/service/src/chat/claim-reconciliation.ts: retain claim subject, attribute, unit, source and supersession separately. Flag a proposed resolution that changes the attribute or unit, and retain unaddressed claims as unknown. Validate on numeric and nonnumeric conflicts; source absence must not become disagreement. This is an advisory prototype hypothesis, with no changes to this frozen campaign.\n\n## Generalization and next experiment\n\nThis is a second first-round suite entry for the same scenario, not completion of three repetitions. Compared with the earlier core result, fixed score rose from 8.5 to 10.0, calls fell from 31 to 8, and duration fell from 55m1s to 28m55s under unchanged tuning. This is natural run variation, not evidence that a change improved the model. Meeting-followup also reconciled conflicting sources successfully; records-intake did not produce its required CSV. The next isolated experiment should test the generic contradiction-versus-unknown instruction across two unrelated reconciliation tasks.\n\n## Performance and fit\n\nPeak VRAM 10626/12227 MiB; process RSS 9872 MiB; whole-system RAM 27913/31865 MiB. Decode 2.7 tokens/sec, prefill 158.58 (perf.gpu, perf.process, perf.systemMemory, perf.derived). Native 0.1.57/CUDA retains 65536 context tokens, f16 KV, one slot and 25 CPU FFN layers, with 39/64 dense FFN layers on GPU. This uses host RAM and CPU; it is not wholly GPU resident. No explicit OOM/context overflow was found; shared GPU memory is not separately measured. Only the expected final shutdown was found in native exit logs.\n\n## Last three same-scenario trials\n\n- [conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/facts.json): pass, fixed 8.5, 3300.6 seconds, 2.84 tokens/sec; underdog-saluki-27b-iq2-mix.\n- [conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2/facts.json): pass, fixed 10, 36.0 seconds, 122.83 tokens/sec; gemma4-12b-q4.\n- [conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-39-24-971Z-sl2p/facts.json): pass, fixed 9.5, 91.5 seconds, 119.86 tokens/sec; gemma4-12b-q4.\n\nThe latest comparison is the earlier Saluki core entry. The two older Gemma4/native 0.1.54 results are historical context.\n" }, { "cellId": "r1-saluki-cuda-productivity-docblocks-theme-roundtrip", "baselineCellId": "r1-saluki-cuda-productivity-docblocks-theme-roundtrip", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T09:39:57.682Z", "finishedAt": "2026-11-04T09:58:59.898Z", "durationMs": 1142216, "success": true, "reason": "theme report is tool-grounded and the saved deck carries the brand (ordered-sections, theme-id, fonts, palette, slide-count, compare-cited, differences-not-whitewashed, recommendation, docblocks-provenance)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1089462 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 1804, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.304908200479539 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1089462 }, { "source": "facts", "path": "outcome.durationMs", "value": 1142216 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 191, "peakRssMb": 11352, "peakCpuPercent": 1207.5, "firstAtMs": 0, "lastAtMs": 1127033 }, "gpu": { "available": true, "sampleCount": 191, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 191, "peakUsedMb": 29642, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 53425, "totalOutputTokens": 1627, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 28139, "todayTokensCached": 28116, "todayTokensOut": 99, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 28139, "totalTokensCached": 28116, "totalTokensOut": 99, "totalCost": 0, "medianOutputTokensPerSec": 2.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.5, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 28139, "cachedInputTokens": 28116, "outputTokens": 99, "cost": 0, "durationMs": 480337, "outputTokensPerSec": 2.529931999816503, "at": "2026-11-04T09:58:49.328Z" }, "lastUpdated": "2026-11-04T09:58:49.328Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T09:58:52.670Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1128498, "meanTokensPerSec": 2.58, "genTokensPerSec": 2.58, "genTokens": 1627, "promptTokensPerSec": 118.81, "peakRssMb": 11352, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28139, "output": 99 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T09:40:03.852Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T09:40:03.763Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wieke": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.3073577880859375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3073577880859375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "theme report is tool-grounded and the saved deck carries the brand (ordered-sections, theme-id, fonts, palette, slide-count, compare-cited, differences-not-whitewashed, recommendation, docblocks-provenance)", "durationMs": 1142216, "timeoutMs": 3746098, "budgetUsedFraction": 0.304908200479539 }, "timing": { "startedAt": "2026-11-04T09:39:57.682Z", "finishedAt": "2026-11-04T09:58:59.898Z", "timeToFirstArtifactMs": 1089462, "timeToLastArtifactWriteMs": 1131646, "timeToFirstTokenMs": 181596, "firstTurnTtftMs": 166355, "timeToFirstToolCallMs": 195048 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "infer_theme_from_file": 1, "apply_inferred_theme": 1, "convert_document": 1, "list_roots": 1, "save_artifact": 1, "inspect_document": 1, "compare_documents": 1, "write_file": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 1134485, "filePath": "docblocks-theme", "bytes": 1804, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } ], "latest": { "filePath": "docblocks-theme", "bytes": 1804, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T09:39:57.687Z] [trial] id=docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr scenario=docblocks-theme-roundtrip model=underdog-saluki-27b-iq2-mix", "[2026-11-04T09:39:57.687Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-docblocks-theme-roundtrip-YnEuF9", "[2026-11-04T09:39:57.687Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "[2026-11-04T09:39:57.687Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T09:39:57.688Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T09:39:57.691Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T09:39:57.695Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T09:39:57.702Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-docblocks-theme-roundtrip-YnEuF9", "[2026-11-04T09:39:57.703Z] [trial] throughput-scaled ceiling: 30m → 62m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T09:40:03.773Z] [trial] daemon spawned pid=29804 port=https://127.0.0.1:53854", "[2026-11-04T09:40:05.095Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T09:40:05.115Z] [trial] meester=sulien", "[2026-11-04T09:40:05.115Z] [trial] running scenario setup hook", "[2026-11-04T09:40:05.582Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T09:40:05.581Z] [trial] scenario setup complete", "[2026-11-04T09:40:05.656Z] [poll] initial digests hard=58af97e28d50a2cb soft=4198a28ec4e21131", "[2026-11-04T09:40:05.778Z] [poll] hard-progress: 58af97e28d50a2cb → 9f364c8045860d8f after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T09:43:15.443Z] [poll] hard-progress: 9f364c8045860d8f → 89d606bd454429eb after 190s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=40,stream=49) sniff=0/0B)", "[2026-11-04T09:43:50.734Z] [poll] hard-progress: 89d606bd454429eb → a6344cad2c4c80d7 after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=82,stream=129) sniff=0/0B)", "[2026-11-04T09:44:37.842Z] [poll] hard-progress: a6344cad2c4c80d7 → fcd928450fe8151c after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=156,stream=271) sniff=0/0B)", "[2026-11-04T09:44:55.562Z] [poll] hard-progress: fcd928450fe8151c → 4e3bc161f74f9b3a after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=162,stream=277) sniff=0/0B)", "[2026-11-04T09:45:42.830Z] [poll] hard-progress: 4e3bc161f74f9b3a → b7b873312683920b after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=248,stream=445) sniff=0/0B)", "[2026-11-04T09:46:24.352Z] [poll] hard-progress: b7b873312683920b → 1ef5fe06afa60dbc after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=301,stream=545) sniff=0/0B)", "[2026-11-04T09:47:17.864Z] [poll] hard-progress: 1ef5fe06afa60dbc → 1b7387c995bf4c51 after 54s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=373,stream=681) sniff=0/0B)", "[2026-11-04T09:50:45.895Z] [poll] hard-progress: 1b7387c995bf4c51 → 6bc0fa302ff4d730 after 208s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=840,stream=1609) sniff=0/0B)" ] }, "metrics": { "process": { "sampleCount": 191, "peakRssMb": 11352, "peakCpuPercent": 1207.5, "firstAtMs": 0, "lastAtMs": 1127033 }, "gpu": { "available": true, "sampleCount": 191, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 191, "peakUsedMb": 29642, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 53425, "totalOutputTokens": 1627, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 28139, "todayTokensCached": 28116, "todayTokensOut": 99, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 28139, "totalTokensCached": 28116, "totalTokensOut": 99, "totalCost": 0, "medianOutputTokensPerSec": 2.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.5, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 28139, "cachedInputTokens": 28116, "outputTokens": 99, "cost": 0, "durationMs": 480337, "outputTokensPerSec": 2.529931999816503, "at": "2026-11-04T09:58:49.328Z" }, "lastUpdated": "2026-11-04T09:58:49.328Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T09:58:52.670Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1128498, "meanTokensPerSec": 2.58, "genTokensPerSec": 2.58, "genTokens": 1627, "promptTokensPerSec": 118.81, "peakRssMb": 11352, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28139, "output": 99 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T09:39:57.682Z", "finishedAt": "2026-11-04T09:58:59.898Z", "timeToFirstArtifactMs": 1089462, "timeToLastArtifactWriteMs": 1131646, "timeToFirstTokenMs": 181596, "firstTurnTtftMs": 166355, "timeToFirstToolCallMs": 195048 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T09:40:03.852Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T09:40:03.763Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wieke": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.3073577880859375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3073577880859375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "scenarioId": "docblocks-theme-roundtrip", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/workspace/brand-theme-alignment/theme-report.md", "artifactBytes": 1812, "artifactKind": "markdown", "axisSource": "Declared judge.axes in evals/src/scenarios/docblocks-theme-roundtrip.ts", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ], "scoreAxes": { "grounding": 5, "candor": 5, "usefulness": 6 }, "meanScore": 5.33, "justification": "Theme values and comparison figures are accurate, and a wording review is actionable. But it explains 26% similarity as expected condensation without evidence and assures no content loss. The deck text actually preserves the source wording. Timing/accessibility degradation is listed but omitted from the recommendation.", "reviewBasis": "Complete report below 20 KiB, source Markdown, exact tool results and saved slide XML text read. This is not a rendered-slide visual review. Model identity visible in paths." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:55.408Z", "reviewId": "018", "cellId": "r1-saluki-cuda-productivity-docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "scenarioId": "docblocks-theme-roundtrip", "status": "scored", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "baselineCellId": "r1-saluki-cuda-productivity-docblocks-theme-roundtrip", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "08ffa8f89d1b922dfa0b1cf2d6bba46f882da9c7c271b084f6b20c717cbfef21", "evidenceSha256": "b0d11d3b7239d9d2c72e402e67512b96a478f0c021a1af73ece56dfe5ce140b8", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/workspace/brand-theme-alignment/theme-report.md", "bytes": 1812, "sha256": "efb76172f12d8bb545ee210e1a5aabe8099a82051849b0dcfbb4315e8a3dd703", "shownBytes": 1812, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr\\workspace\\brand-theme-alignment\\source\\quarterly-review.md", "bytes": 443, "sha256": "70a119718fdfd89aba5d0dd2bdc2e7df8fb45f749afc71d56bb5980e1fb030a8", "shownBytes": 443, "truncated": false } ], "scoreAxes": { "grounding": 6, "candor": 6, "usefulness": 5 }, "justification": "Theme id, fonts, hexes and 4 slides match the brief. But \"Halvard Brand\", \"sans-serif fallback\", and \"condensed rendering... not a content loss\" are unverified embellishments. The 26% text similarity and degraded items are listed plainly, though partly explained away. The recommendation is a vague \"review wording once\", not a ship/hold decision.", "limitations": "The tool responses weren't shown, so I couldn't check the comparison figures or whether \"Halvard Brand\" and the font fallbacks came from tools. The deck's actual text wasn't visible either, so the \"condensed\" claim couldn't be verified. This is a source-only review.", "meanScore": 5.666666666666667, "durationMs": 21479, "costUsd": 0.023273999999999996, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3925, "cache_read_input_tokens": 0, "output_tokens": 757, "output_tokens_details": { "thinking_tokens": 406 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3925, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 757, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 3925, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3925 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 757, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 3925, "webSearchRequests": 0, "costUSD": 0.023273999999999996, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 406, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3925, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - docblocks-theme-roundtrip / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 19m 2s; budget used: 30.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1089462`; `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1804,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.304908200479539`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 1089462`; `facts.outcome.durationMs: 1142216` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11352 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29642 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.58 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 53425 + 1627 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 31% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1089462`\n- `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1804,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.304908200479539`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 1089462`\n- `facts.outcome.durationMs: 1142216`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1089462`; `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1804,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.304908200479539`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 1089462`; `facts.outcome.durationMs: 1142216`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nRead the complete 1812-byte declared theme-report.md, its source, and the tool results. The three exact axes from evals/src/scenarios/docblocks-theme-roundtrip.ts are scored separately from fixed composite 10.0:\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| grounding | 5 | No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication. |\n| candor | 5 | Differences the comparison found are stated plainly rather than smoothed over. |\n| usefulness | 6 | The recommendation is a decision someone could act on this week. |\n\n**Advisory mean: 5.33/10, codex-inline/current-session.**\n\nTheme values and comparison figures are accurate, and a wording review is actionable. But it explains 26% similarity as expected condensation without evidence and assures no content loss. The deck text actually preserves the source wording. Timing/accessibility degradation is listed but omitted from the recommendation.\n\nThe four saved slide XML text bodies preserve the source headings and sentences. The comparison's 26% figure therefore should not be casually interpreted as shortening; the model had inspect_document text available. The report correctly quotes that metric, but its causal explanation is unsupported. This is a report review and text inspection, not a rendered-slide visual quality score.\n\n## Positives\n\n- All six required DocBlocks operations succeeded, and the real 19,853-byte PowerPoint carries the brand theme (toolUse.byTool; outcome.reason; sniff.latest.score: 9/9). Unlike a narrated conversion, these actions have tool provenance and a persisted binary artifact.\n- The report accurately gives both fonts, both required colors, four slides and the tool's comparison figures (sniff.latest.signals; recording/chat-events.jsonl). No fixed-rubric red flags or missing roles were recorded (toolUse.redFlags; team.missingExpectedRoles).\n\n## Negatives\n\n- Two write_file attempts were needed (toolUse.byTool.write_file: 2). The first was refused as ungrounded even for the exact returned fonts and comparison numbers; the second was admitted after the configured one-refusal limit. The false missing-evidence flags and the genuinely unsupported condensation claim are distinct problems (daemon.log at 09:50:45Z and 09:58:07Z; advisory-tool-evidence.json).\n- Elapsed time was 1,142,216 ms, 19m2s, at 2.58 decode tokens/sec (outcome.durationMs; perf.derived). Fixed score 10.0 does not imply an error-free engine session: there were two native launches and a transient unreachable chat-completions endpoint, detailed below (campaign row result.engineContext.launches; recording/chat-events.jsonl).\n\n## Native reliability observation\n\nAt 09:50:48.869Z the chat stream reported “/v1/chat/completions unreachable ... fetch failed.” At 09:50:49.353Z the supervisor logged yielding the idle engine to a waiting memory request, then logged the old process exiting with code 3221226505, expected=true, expectedReason=memory-pressure. A new process launched at 09:50:51.539Z and loaded by 09:50:58.622Z; the standard queued continuation completed successfully. Preserve all of these facts: the endpoint error precedes the eviction label, and the nonzero exit cannot be described as an uninterrupted healthy session. The available log does not prove whether the first failure caused or resulted from memory coordination. No explicit CUDA OOM or context overflow was found. The campaign did not stop, and no manual restart or capability retry was made.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/de/designer/versions/1.1.0/about.md, add: “Separate measured comparison values from explanations. Label an explanation as unverified unless inspection supports it, and include a concrete follow-up for every reported degradation.” Test on another document conversion and a source-comparison report. Preserve exact returned metrics; do not teach this role a fixture-specific explanation for a percentage.\n\n## Strategic experiments — not applied\n\nIn packages/core/src/grounding/factual-writing.ts, add explicit evidence recognition for successful read-only infer_theme_from_file, inspect_document and compare_documents results, preserving source identity through packages/service/src/chat/evidence-ledger.ts. Current EVIDENCE_TOOLS omits them and the third-party lookup-verb rule does not recognize infer/inspect/compare. Validate that returned facts become usable evidence while unexecuted calls and the model's own prose do not. The first refusal provides a concrete cross-format evidence-registration bug; do not weaken grounding or its refusal limits to hide it.\n\nIn packages/inference/src/providers/native/supervisor.ts, investigate ordering of active-request tracking, yieldForWaitingCapacity and nonzero-exit classification. Capture the first transport-failure timestamp and stop-intent timestamp separately so a later expected eviction cannot obscure an earlier failure. Reproduce with isolated alternating inference/memory claims after this campaign. No cause or fix is established from a single automatically recovered incident.\n\n## Generalization and next experiment\n\nThis trial completed the real conversion chain, unlike the spreadsheet-model trial's unfinished HTML write. Petshop also exercised memory-driven engine eviction, making capacity transitions relevant beyond document tools, though it did not establish the same transport failure. Meeting-followup provides a successful report-grounding baseline. Test the evidence registration change on document inspection plus an unrelated read-only connector report, and investigate native handoff separately so effects remain attributable.\n\n## Performance and fit\n\nPeak VRAM 10638/12227 MiB; process RSS 11352 MiB; whole-system RAM 29642/31865 MiB. Decode 2.58 tokens/sec, prefill 118.81 (perf.gpu, perf.process, perf.systemMemory, perf.derived). Native 0.1.57/CUDA retains 65536 context tokens, f16 KV, one slot and 25 CPU FFN layers, with 39/64 dense FFN layers on GPU. This uses host RAM and CPU; it is not wholly GPU resident. No explicit OOM/context overflow was found; shared GPU memory is not separately measured. Two launches and the recovered transport failure are documented above; do not collapse that into “no native failures.”\n\n## Last three same-scenario trials\n\n- [docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/facts.json): fail, fixed 5.9, 512.1 seconds, 115.1 tokens/sec; gemma4-12b-q4.\n- [docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao/facts.json): fail, fixed 5.9, 550.3 seconds, 116.62 tokens/sec; gemma4-12b-q4.\n- [docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-41-37-082Z-iye8/facts.json): fail, fixed 3.8, 291.9 seconds, 115.21 tokens/sec; gemma4-12b-q4.\n\nAll three older Gemma4/native 0.1.54 runs exhausted repairs, with missing DocBlocks inference/provenance; this trial completed the required tool chain. Model, source and native identity differ, so this is not a controlled improvement claim.\n" }, { "cellId": "r1-saluki-cuda-productivity-craftbook-powerpoint-deck", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-powerpoint-deck", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T10:01:30.181Z", "finishedAt": "2026-11-04T11:14:43.453Z", "durationMs": 4393272, "success": false, "reason": "hit hard ceiling (4370447ms) — last hard progress 1409s before the end; the deliverable never closed", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "failureMode": "timeout", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-powerpoint-deck", "score": 12, "bytes": 4024, "failReason": "deliverables/d-day.pptx is 0 bytes, need ≥ 1000", "deliverableMissing": true, "milestones": 4 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "timeout" }, { "source": "facts", "path": "outcome.reason", "value": "hit hard ceiling (4370447ms) — last hard progress 1409s before the end; the deliverable never closed" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1082143 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-powerpoint-deck", "bytes": 4024, "score": 12, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" } } ] }, "quality": { "score": 5.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 57.1% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1082143 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-powerpoint-deck", "bytes": 4024, "score": 12, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1082143 }, { "source": "facts", "path": "outcome.durationMs", "value": 4393272 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.6, "band": "framework-gap" }, "facts": { "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 739, "peakRssMb": 14231, "peakCpuPercent": 1219.2, "firstAtMs": 0, "lastAtMs": 4375972 }, "gpu": { "available": true, "sampleCount": 739, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 739, "peakUsedMb": 31332, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 46320, "totalOutputTokens": 7321, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 35229, "todayTokensCached": 33609, "todayTokensOut": 2511, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 35229, "totalTokensCached": 33609, "totalTokensOut": 2511, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11389, "cachedInputTokens": 11319, "outputTokens": 48, "cost": 0, "durationMs": 436332, "outputTokensPerSec": 2.825479792018236, "at": "2026-11-04T10:43:58.030Z" }, "lastUpdated": "2026-11-04T10:43:58.030Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T11:14:34.003Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4377508, "meanTokensPerSec": 2.82, "genTokensPerSec": 2.82, "genTokens": 7321, "promptTokensPerSec": 159.68, "peakRssMb": 14231, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35229, "output": 2511 } } } }, "craftbook": { "craftbookId": "powerpoint-deck", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 6, "activationEvents": 3, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "powerpoint-grounding-eval/1", "stepId": "research", "activatedAt": "2026-11-04T10:01:38.140Z", "completedAt": "2026-11-04T10:19:32.554Z", "ms": 1074414, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "09ef641e-527a-4152-ac18-e0c887678d04" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "outline", "activatedAt": "2026-11-04T10:19:32.557Z", "completedAt": "2026-11-04T10:36:37.382Z", "ms": 1024825, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "5326d3fe-d274-416b-8d5a-8c978c41702c" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "write", "activatedAt": "2026-11-04T10:36:37.383Z", "completedAt": "2026-11-04T10:43:58.000Z", "ms": 440617, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "3ef246bb-29ae-45f9-bb9b-b985966e9c57" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "review", "activatedAt": "2026-11-04T10:43:58.001Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "6aa563c6-0651-4acf-bae3-d86c1f80c1e6" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T10:01:36.156Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T10:01:36.056Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1024825 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "joko": 1, "martina": 1, "simone": 1, "zofia": 1 }, "perTask": { "powerpoint-grounding-eval/1": 4 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6666666666666666, "maxMessages": 2, "maxContextFill": 0.150238037109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.150238037109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "timeout", "reason": "hit hard ceiling (4370447ms) — last hard progress 1409s before the end; the deliverable never closed", "durationMs": 4393272, "timeoutMs": 4370447, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-04T10:01:30.181Z", "finishedAt": "2026-11-04T11:14:43.453Z", "timeToFirstArtifactMs": 1082143, "timeToLastArtifactWriteMs": 2547849, "timeToFirstTokenMs": 86756, "firstTurnTtftMs": 70269, "timeToFirstToolCallMs": 122470 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Researcher", "Planner", "Copywriter", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 19, "byTool": { "read_file": 6, "search": 1, "write_artifact": 4, "read_artifact": 5, "write_file": 1, "write_task_note": 1, "advance_task_step": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 8296, "filePath": "craftbook-powerpoint-deck", "bytes": 0, "score": 1, "scoreMax": 21, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/brief.md" }, { "atMs": 126949, "filePath": "craftbook-powerpoint-deck", "bytes": 0, "score": 2, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 1082432, "filePath": "craftbook-powerpoint-deck", "bytes": 4024, "score": 4, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 2111671, "filePath": "craftbook-powerpoint-deck", "bytes": 4024, "score": 6, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 2467133, "filePath": "craftbook-powerpoint-deck", "bytes": 4024, "score": 12, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" } ], "latest": { "filePath": "craftbook-powerpoint-deck", "bytes": 4024, "score": 12, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T10:01:30.185Z] [trial] id=craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31 scenario=craftbook-powerpoint-deck model=underdog-saluki-27b-iq2-mix", "[2026-11-04T10:01:30.185Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "[2026-11-04T10:01:30.185Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T10:01:30.185Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-powerpoint-deck-fFekPU", "[2026-11-04T10:01:30.185Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T10:01:30.187Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T10:01:30.191Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T10:01:30.197Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-powerpoint-deck-fFekPU", "[2026-11-04T10:01:30.197Z] [trial] throughput-scaled ceiling: 35m → 73m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T10:01:36.056Z] [trial] daemon spawned pid=41216 port=https://127.0.0.1:64909", "[2026-11-04T10:01:37.686Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T10:01:37.706Z] [trial] meester=celestine", "[2026-11-04T10:01:37.706Z] [trial] running scenario setup hook", "[2026-11-04T10:01:38.341Z] [trial] scenario setup complete", "[2026-11-04T10:01:38.343Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T10:01:38.395Z] [poll] initial digests hard=a405390ee9ab0ea4 soft=2500bf3f349f3348", "[2026-11-04T10:01:38.520Z] [poll] hard-progress: a405390ee9ab0ea4 → 6677751c5aadcf1a after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T10:01:44.465Z] [poll] hard-progress: 6677751c5aadcf1a → b0b8aa96e5fe3606 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T10:03:37.173Z] [poll] hard-progress: b0b8aa96e5fe3606 → b508438208c03009 after 113s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=94,stream=93) sniff=2/0B)", "[2026-11-04T10:04:13.486Z] [poll] hard-progress: b508438208c03009 → 6707171a1bea9bfb after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=163,stream=170) sniff=2/0B)", "[2026-11-04T10:11:50.369Z] [poll] hard-progress: 6707171a1bea9bfb → ead99ab98d12af2b after 457s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=1366,stream=2389) sniff=2/0B)", "[2026-11-04T10:12:49.625Z] [poll] hard-progress: ead99ab98d12af2b → edbe71b468af6e70 after 59s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1478,stream=2508) sniff=2/0B)", "[2026-11-04T10:19:32.669Z] [poll] hard-progress: edbe71b468af6e70 → b8feef65c449d0e6 after 403s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=2533,stream=4539) sniff=4/4024B)", "[2026-11-04T10:19:38.543Z] [poll] hard-progress: b8feef65c449d0e6 → 6e8a33290b6d7589 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=2533,stream=4539) sniff=4/4024B)", "[2026-11-04T10:21:01.607Z] [poll] hard-progress: 6e8a33290b6d7589 → 0ed023c8369f7640 after 83s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=2592,stream=4608) sniff=4/4024B)" ] }, "metrics": { "process": { "sampleCount": 739, "peakRssMb": 14231, "peakCpuPercent": 1219.2, "firstAtMs": 0, "lastAtMs": 4375972 }, "gpu": { "available": true, "sampleCount": 739, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 739, "peakUsedMb": 31332, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 46320, "totalOutputTokens": 7321, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 35229, "todayTokensCached": 33609, "todayTokensOut": 2511, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 35229, "totalTokensCached": 33609, "totalTokensOut": 2511, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11389, "cachedInputTokens": 11319, "outputTokens": 48, "cost": 0, "durationMs": 436332, "outputTokensPerSec": 2.825479792018236, "at": "2026-11-04T10:43:58.030Z" }, "lastUpdated": "2026-11-04T10:43:58.030Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T11:14:34.003Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4377508, "meanTokensPerSec": 2.82, "genTokensPerSec": 2.82, "genTokens": 7321, "promptTokensPerSec": 159.68, "peakRssMb": 14231, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 35229, "output": 2511 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T10:01:30.181Z", "finishedAt": "2026-11-04T11:14:43.453Z", "timeToFirstArtifactMs": 1082143, "timeToLastArtifactWriteMs": 2547849, "timeToFirstTokenMs": 86756, "firstTurnTtftMs": 70269, "timeToFirstToolCallMs": 122470 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 6, "activationEvents": 3, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "powerpoint-grounding-eval/1", "stepId": "research", "activatedAt": "2026-11-04T10:01:38.140Z", "completedAt": "2026-11-04T10:19:32.554Z", "ms": 1074414, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "09ef641e-527a-4152-ac18-e0c887678d04" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "outline", "activatedAt": "2026-11-04T10:19:32.557Z", "completedAt": "2026-11-04T10:36:37.382Z", "ms": 1024825, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "5326d3fe-d274-416b-8d5a-8c978c41702c" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "write", "activatedAt": "2026-11-04T10:36:37.383Z", "completedAt": "2026-11-04T10:43:58.000Z", "ms": 440617, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "3ef246bb-29ae-45f9-bb9b-b985966e9c57" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "review", "activatedAt": "2026-11-04T10:43:58.001Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "6aa563c6-0651-4acf-bae3-d86c1f80c1e6" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T10:01:36.156Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T10:01:36.056Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1024825 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "joko": 1, "martina": 1, "simone": 1, "zofia": 1 }, "perTask": { "powerpoint-grounding-eval/1": 4 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6666666666666666, "maxMessages": 2, "maxContextFill": 0.150238037109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.150238037109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "scenarioId": "craftbook-powerpoint-deck", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-04T12:19:48.736Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "artifactBytes": 2354, "artifactSha256": "d1d494f8904a0462aeb19b8fa6fd39fac72008a5b4b44e1bf41b8929795e71ae", "artifactKind": "markdown", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/po/powerpoint-deck/versions/1.7.19/test.json", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ], "scoreAxes": { "source-acquisition": 8, "outline-parity": 8, "grounding": 7, "convertibility": 6 }, "meanScore": 7.25, "justification": "The source was read before outlining; all seven slide titles match the outline, and metrics/actions stay faithful to the brief. Repeated citations and caveats make the source wordy. The approval request loses its proposal label, and “the gap the next actions address” implies more support than the brief supplies.", "reviewBasis": "Re-read complete final captured Markdown and verified unchanged hash against the source/outline review. No rendered or binary presentation exists. Final score describes the declared Markdown target only.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "status": "present", "bytes": 4040 }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "status": "present", "bytes": 4056 }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "status": "present", "bytes": 2354 }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/artifacts/powerpoint-grounding-eval/tasks/1/deck.pptx", "status": "noArtifact", "reason": "Required binary absent at terminal verdict. No binary scores invented." }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/workspace/powerpoint-grounding-eval/deliverables/d-day.pptx", "status": "noArtifact", "reason": "Required binary absent at terminal verdict. No binary scores invented." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:57:55.136Z", "reviewId": "019", "cellId": "r1-saluki-cuda-productivity-craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "scenarioId": "craftbook-powerpoint-deck", "status": "scored", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-powerpoint-deck", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b20af1ba34fd1b509de0d205fc8c0b2496b2940f308b563ebff8d7f8dfaf8ff7", "evidenceSha256": "cb8165bb801cc87395c700761972c5477ea8b4fa361625149873a82119a28153", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "bytes": 2354, "sha256": "d1d494f8904a0462aeb19b8fa6fd39fac72008a5b4b44e1bf41b8929795e71ae", "shownBytes": 2354, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "bytes": 4040, "sha256": "d42ac226eac0c9d9dec8ed539839da86e723f88a97e1fed1fbc284c85334563a", "shownBytes": 4040, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "bytes": 4056, "sha256": "bbf7ba82bebf82bbc6112503324dcddc37e5da56cf099cbd48039583bb2680cb", "shownBytes": 4056, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31\\workspace\\powerpoint-grounding-eval\\source\\brief.md", "bytes": 381, "sha256": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "shownBytes": 381, "truncated": false } ], "scoreAxes": { "source-acquisition": 9, "outline-parity": 10, "grounding": 8, "convertibility": 7 }, "justification": "sources.md records the brief's path, boundary, a fact ledger and honest no-external-research rationale. The 7 outline titles match deck.md H1s exactly, in order. All figures match brief.md. Small inferences (\"gap the next actions address\"). Inline citations on every bullet clutter slides.", "limitations": "No PPTX was produced or inspected (declared paths missing), so conversion readiness is judged from the Markdown only. The run was terminal, with no rendering evidence. The evidence does not show whether the later workflow steps ran.", "meanScore": 8.5, "durationMs": 16993, "costUsd": 0.039813999999999995, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 8110, "cache_read_input_tokens": 0, "output_tokens": 737, "output_tokens_details": { "thinking_tokens": 411 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 8110, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 737, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 8110, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 8110 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 737, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 8110, "webSearchRequests": 0, "costUSD": 0.039813999999999995, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 411, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 8110, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-powerpoint-deck / underdog-saluki-27b-iq2-mix\n\n**Composite: 4.6 / 10** (band: framework-gap)\n**Outcome: timeout** (duration: 73m 13s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (4370447ms) — last hard progress 1409s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 1082143`; `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":4024,\"score\":12,\"scoreMax\":21,\"signals\":[],\"failReason\":\"task sourced from craftbook powerpoint-deck has not finished: its gates ...` |\n| Output quality (25%) | 5.7 / 10 | The latest scenario gate reached 57.1% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1082143`; `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":4024,\"score\":12,\"scoreMax\":21,\"signals\":[],\"failReason\":\"task sourced from craftbook powerpoint-deck has not finished: its gates ...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 1082143`; `facts.outcome.durationMs: 4393272` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 5.7 + 0.2 * 2.5 + 0.15 * 10 = **4.6**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14231 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31332 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.82 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 46320 + 7321 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 6 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1025s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.67 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"timeout\"`\n- `facts.outcome.reason: \"hit hard ceiling (4370447ms) — last hard progress 1409s before the end; the deliverable never closed\"`\n- `facts.timing.timeToFirstArtifactMs: 1082143`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":4024,\"score\":12,\"scoreMax\":21,\"signals\":[],\"failReason\":\"task sourced from craftbook powerpoint-deck has not finished: its gates ...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 57.1% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 1082143`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":4024,\"score\":12,\"scoreMax\":21,\"signals\":[],\"failReason\":\"task sourced from craftbook powerpoint-deck has not finished: its gates ...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 19`\n- `facts.timing.timeToFirstArtifactMs: 1082143`\n- `facts.outcome.durationMs: 4393272`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (4370447ms) — last hard progress 1409s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 1082143`; `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":4024,\"score\":12,\"scoreMax\":21,\"signals\":[],\"failReason\":\"task sourced from craftbook powerpoint-deck has not finished: its gates ...`\n- Quality: The latest scenario gate reached 57.1% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1082143`; `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":4024,\"score\":12,\"scoreMax\":21,\"signals\":[],\"failReason\":\"task sourced from craftbook powerpoint-deck has not finished: its gates ...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 1082143`; `facts.outcome.durationMs: 4393272`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Final advisory inline judge\n\nRe-read the captured 2354-byte slide Markdown after termination and verified its SHA-256 matches the earlier provisional review. The declared judge target exists, so its four axes remain scored even though the binary deliverables are absent. Both tasks/1/deck.pptx in artifacts and deliverables/d-day.pptx in the workspace are noArtifact, with no invented binary quality scores. Fixed composite remains 4.6.\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| source-acquisition | 8 | The workflow read and cited the exact supplied source before outlining. |\n| outline-parity | 8 | Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order. |\n| grounding | 7 | Every metric and action comes from source/brief.md and no extra metric was invented. |\n| convertibility | 6 | One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion. |\n\n**Advisory mean: 7.25/10, codex-inline/current-session.**\n\nThe source was read before outlining; all seven slide titles match the outline, and metrics/actions stay faithful to the brief. Repeated citations and caveats make the source wordy. The approval request loses its proposal label, and “the gap the next actions address” implies more support than the brief supplies.\n\nThese scores describe source preparation. No PowerPoint was converted or previewed, so they do not establish presentation delivery or rendered quality. The provisional entry has been finalized under trials in codex-inline-judge.json; its original snapshot remains in advisory-progress-review.json for audit.\n\n## Positives\n\n- The real craftbook advanced through research, outline and writing, with all three completed phases recorded (craftbook.craftbookTasks: 1; continuity.steps.perStep). The supplied source was read before the source packet and outline, and the final Markdown carries all seven outline titles in order.\n- Source packet, outline and slide Markdown were committed, covering 12/21 deterministic checks (sniff.latest; timing.timeToFirstArtifactMs: 1082143). No missing expected roles, fixed-rubric red flags or user interventions were recorded (team.missingExpectedRoles; toolUse.redFlags; autoAnswer.total).\n\n## Negatives\n\n- The trial stopped in review after 4,393,272 ms (73m13s), with no terminal phase or PowerPoint output (outcome.durationMs; craftbook.reachedTerminal: false; continuity.steps.perStep). None of convert_document, preview_document, save_artifact or inspect_document completed (toolUse.byTool). Their absence does not establish a converter failure: execution never reached the conversion phase.\n- Research and outline each incurred a refused write followed by another full write, helping consume 17m54s and 17m5s respectively (toolUse.byTool.write_artifact: 4; continuity.steps.perStep; recording/chat-events.jsonl). Some refusals concerned proposed slide wording and a chosen slide count, rather than fabricated source metrics. Keep this distinction visible when evaluating grounding behavior.\n- The reviewer generated 15,004 reasoning characters through 11:09:03Z, then streamed 2822 incomplete write_artifact argument characters until the 11:14:33Z cutoff (advisory-generation-evidence.json; outcome.failureMode: timeout). No review artifact was committed. Fixed performance totals report 7321 completed-request output tokens; the unfinished request separately reached 3912. The reviewer's persisted transcript was incomplete, so zero persisted tool calls would be misleading (log.txt; toolUse.totalToolCalls: 19).\n\n## Tactical experiment — not applied\n\nFor a later A/B, reduce tuning.reasoning.thinkingBudget and the effective thinking-general/thinking-coding profile overrides from 4096 to 1024 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json. Preserve output caps, gates, sampling and timeouts. Measure whether the reviewer completes its first mutation sooner without weakening grounding, using this workflow, another document workflow and a coding review. The late unfinished write and roughly 2.8 tokens/sec justify the hypothesis, not a claim that it is already a fix.\n\n## Strategic experiments — not applied\n\nIn packages/core/src/local-loop/constrained-turn.ts, investigate a task-wide first-deliverable reasoning allocation that reserves time for downstream production after source preparation and review. Preserve every required phase and workspace-write boundary; do not declare draft Markdown a substitute for the requested binary. Measure time in each phase and actual delivered files across document and multi-step data tasks.\n\nIn packages/service/src/chat/evidence-ledger.ts and packages/core/src/grounding/citations.ts, investigate typed distinctions between source facts, quoted proposed wording and user-authorized design choices. A slide count chosen within the requested range should not require a source citation as a historical fact. Preserve checks against invented metrics, dates and attribution; test both genuine fabrications and harmless design proposals. This recommendation is grounded in recorded refusals, not a proposal to relax the current evaluation.\n\n## Generalization and next experiment\n\nThe spreadsheet workflow likewise committed a useful early artifact then failed before its final output. Incident-postmortem and the research brief also timed out during tool-argument assembly, while the DocBlocks theme task successfully saved a real PowerPoint through a shorter conversion sequence. The recurring issue is time to commit required output across phases, not demonstrated inability to load or a proven broken converter. Test the reasoning-budget change across unrelated task types in a separate campaign; leave this failure and its gates intact.\n\n## Performance and fit\n\nRecorded decode 2.82 tokens/sec; prefill 159.68. Peak VRAM 10638/12227 MiB, process RSS 14231 MiB, whole-system RAM 31332/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Launch retained native 0.1.57/CUDA, 65536 context, one slot, f16 KV and 25 CPU FFN layers, with dense FFN of 39/64 layers on GPU. No explicit CUDA OOM, context overflow, unexpected native exit or transport error found in this trial; native shutdown followed the verdict normally. CPU/host-RAM placement is intentional; shared GPU memory and paging are not separately measured. Whole-system RAM approached capacity, but includes other processes and does not establish paging as the cause of the failure.\n\n## Last three same-scenario trials\n\n- [craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/facts.json): pass, fixed 8.5, 153.4 seconds, recorded decode 125.67 tokens/sec.\n- [craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/facts.json): pass, fixed 8.5, 169.7 seconds, recorded decode 121.38 tokens/sec.\n- [craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-04-16-817Z-rff4/facts.json): pass, fixed 9, 153.9 seconds, recorded decode 122.23 tokens/sec.\n\nAll three older Gemma4 12B/native 0.1.54 trials passed. Model, native release and source identity differ, so the current failure is historical contrast, not a controlled regression attributable to 0.1.57.\n" }, { "cellId": "r1-saluki-cuda-productivity-wikipedia-research-brief", "baselineCellId": "r1-saluki-cuda-productivity-wikipedia-research-brief", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T11:17:39.778Z", "finishedAt": "2026-11-04T12:02:02.961Z", "durationMs": 2663183, "success": false, "reason": "no real progress for 1877s — model busy but not delivering (hard digest stuck at c8213895a0cdf357)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "wikipedia-research-brief", "score": 0, "bytes": 0 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1877s — model busy but not delivering (hard digest stuck at c8213895a0cdf357)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "quality": { "score": 0, "ruleId": "Q_NO_PRODUCED_OUTPUT", "summary": "No authored artifact event or scored sniff output was observed.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6093616968699083 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 2663183 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3, "band": "framework-gap" }, "facts": { "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 449, "peakRssMb": 10322, "peakCpuPercent": 1208.7, "firstAtMs": 0, "lastAtMs": 2646654 }, "gpu": { "available": true, "sampleCount": 449, "peakUtilPercent": 100, "peakMemUsedMb": 10624, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 449, "peakUsedMb": 28489, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 10933, "totalOutputTokens": 1968, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T12:01:54.455Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2648171, "meanTokensPerSec": 2.9, "genTokensPerSec": 2.9, "genTokens": 1968, "promptTokensPerSec": 161.38, "peakRssMb": 10322, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T11:17:45.920Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T11:17:45.827Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "elian": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0751953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0751953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1877s — model busy but not delivering (hard digest stuck at c8213895a0cdf357)", "durationMs": 2663183, "timeoutMs": 4370447, "budgetUsedFraction": 0.6093616968699083 }, "timing": { "startedAt": "2026-11-04T11:17:39.778Z", "finishedAt": "2026-11-04T12:02:02.961Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 43330, "firstTurnTtftMs": 31578, "timeToFirstToolCallMs": 652729 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 3, "byTool": { "write_file": 1, "search_wikipedia": 1, "read_wikipedia_sources": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 2 }, "sniff": { "progression": [], "latest": null }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T11:17:39.782Z] [trial] id=wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd scenario=wikipedia-research-brief model=underdog-saluki-27b-iq2-mix", "[2026-11-04T11:17:39.782Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-wikipedia-research-brief-AZ9KTQ", "[2026-11-04T11:17:39.782Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "[2026-11-04T11:17:39.782Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T11:17:39.782Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T11:17:39.785Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T11:17:39.788Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T11:17:39.795Z] [trial] throughput-scaled ceiling: 35m → 73m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T11:17:39.794Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-wikipedia-research-brief-AZ9KTQ", "[2026-11-04T11:17:45.813Z] [trial] daemon spawned pid=24012 port=https://127.0.0.1:50493", "[2026-11-04T11:17:47.218Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T11:17:47.236Z] [trial] meester=senna", "[2026-11-04T11:17:47.236Z] [trial] running scenario setup hook", "[2026-11-04T11:17:47.718Z] [trial] scenario setup complete", "[2026-11-04T11:17:47.720Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T11:17:47.800Z] [poll] initial digests hard=445e88b03a89afbf soft=9d0f0063f42f3057", "[2026-11-04T11:17:47.942Z] [poll] hard-progress: 445e88b03a89afbf → 69fc3ac819392b00 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T11:28:36.937Z] [poll] hard-progress: 69fc3ac819392b00 → 3d215e25aac34105 after 649s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=1752,stream=3486) sniff=0/0B)", "[2026-11-04T11:30:12.519Z] [poll] hard-progress: 3d215e25aac34105 → 2704abab612dacdc after 96s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=1878,stream=3610) sniff=0/0B)", "[2026-11-04T11:30:36.494Z] [poll] hard-progress: 2704abab612dacdc → c8213895a0cdf357 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=1915,stream=3648) sniff=0/0B)", "[2026-11-04T12:01:53.993Z] [poll] no-progress (hard): no real progress for 1877s (threshold=1873s); model busy but not delivering — failing trial", "[2026-11-04T12:01:54.511Z] [capture] dumped 1 session(s)", "[2026-11-04T12:01:54.514Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T12:01:54.515Z] [capture] INCOMPLETE TRANSCRIPT for 1 session(s) — elian: 0 persisted vs 3 recorded. Assistant turns commit at turn END, so a mid-turn termination loses them. Triage from session-telemetry.json and daemon.log, NOT from the empty transcript.", "[2026-11-04T12:01:54.562Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)" ] }, "metrics": { "process": { "sampleCount": 449, "peakRssMb": 10322, "peakCpuPercent": 1208.7, "firstAtMs": 0, "lastAtMs": 2646654 }, "gpu": { "available": true, "sampleCount": 449, "peakUtilPercent": 100, "peakMemUsedMb": 10624, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 449, "peakUsedMb": 28489, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 10933, "totalOutputTokens": 1968, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T12:01:54.455Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2648171, "meanTokensPerSec": 2.9, "genTokensPerSec": 2.9, "genTokens": 1968, "promptTokensPerSec": 161.38, "peakRssMb": 10322, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T11:17:39.778Z", "finishedAt": "2026-11-04T12:02:02.961Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 43330, "firstTurnTtftMs": 31578, "timeToFirstToolCallMs": 652729 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T11:17:45.920Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T11:17:45.827Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "elian": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0751953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0751953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-wikipedia-research-brief", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "scenarioId": "wikipedia-research-brief", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/workspace/analytical-engine-research-brief/research-brief.md", "reason": "Initial write rejected as ungrounded; subsequent write arguments incomplete at no-progress watchdog. Declared artifact absent; no qualitative scores invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:04.857Z", "reviewId": "020", "cellId": "r1-saluki-cuda-productivity-wikipedia-research-brief", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "scenarioId": "wikipedia-research-brief", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "baselineCellId": "r1-saluki-cuda-productivity-wikipedia-research-brief", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "8460f284dd9b51ac146fc5db48b77f2817af12cfd30fdb9ad96f050e6b1abc99", "evidenceSha256": "d2a130f3ce085f732dfa6c97687034438d099d9381a06333f140f24a1f65643b", "artifactFiles": [], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd\\workspace\\analytical-engine-research-brief\\mocks\\services.md", "bytes": 886, "sha256": "4e5e67d58565bae7556a857a70daeccf632c69b60e3f581d5f46628c874cde66", "shownBytes": 886, "truncated": false } ], "scoreAxes": {}, "justification": "The declared deliverable workspace/analytical-engine-research-brief/research-brief.md was absent at closeout. The artifacts list is empty and the missing path is flagged. The evidence has no tool-call logs or transcript, so the reason for the missing file can't be determined. Only the mock services context file was shown.", "limitations": "Partial evidence only: no transcript, tool-call logs or Wikipedia responses. I can't tell whether the searches ran or why the file wasn't written. The disposition is runner-terminal, so I draw no model success or failure conclusion. No scores were assigned.", "meanScore": null, "durationMs": 20517, "costUsd": 0.01148, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2074, "cache_read_input_tokens": 700, "output_tokens": 311, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2074, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 311, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2074, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2074 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 311, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2074, "webSearchRequests": 0, "costUSD": 0.01148, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2074, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - wikipedia-research-brief / underdog-saluki-27b-iq2-mix\n\n**Composite: 3.0 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 44m 23s; budget used: 60.9%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1877s — model busy but not delivering (hard digest stuck at c8213895a0cdf357)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null` |\n| Output quality (25%) | 0 / 10 | No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.6093616968699083`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 2663183` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 0 + 0.2 * 7.5 + 0.15 * 10 = **3.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10322 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10624 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28489 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.9 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 10933 + 1968 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 8% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1877s — model busy but not delivering (hard digest stuck at c8213895a0cdf357)\"`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: null`\n\n### Output quality - Q_NO_PRODUCED_OUTPUT\n\nNo authored artifact event or scored sniff output was observed.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: null`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.6093616968699083`\n- `facts.toolUse.totalToolCalls: 3`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 2663183`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1877s — model busy but not delivering (hard digest stuck at c8213895a0cdf357)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null`\n- Quality: No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.6093616968699083`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 2663183`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory outcome\n\nThe required research-brief.md is absent. Record noArtifact and no scores for the declared synthesis, grounding, citationQuality or calibration axes. The mocked services description and shared library document are seeded support files, not delivered research. Fixed composite remains 3.0. The initial rejected write and the later incomplete argument stream are evidence about execution, not usable artifacts.\n\n## Positives\n\n- The grounding guard rejected the initial unsupported draft before it was saved (toolUse.byTool.write_file: 1; timing.timeToFirstArtifactMs: null; recording/chat-events.jsonl). That attempt included dates and publication details absent from the permitted source cards, despite fabricated source-id citations.\n- Recovery did call the local search and source-card reader successfully (toolUse.byTool.search_wikipedia: 1; read_wikipedia_sources: 1). All signed cards were returned without truncation. No live-retrieval call, fixed-rubric red flag or missing expected role was recorded (toolUse.byTool; toolUse.redFlags; team.missingExpectedRoles). The local research service was available.\n\n## Negatives\n\n- The first call tried to write a full report before either required source lookup (toolUse.byTool; recording/chat-events.jsonl at 11:28:32Z). Native-provider logs show why this cannot be attributed only to the model ignoring instructions: at 11:18:02Z the runtime selected “write_file-only surface, thinking disabled.” The prompt explicitly required search followed by reading first. This is a concrete ordering conflict in the hosting policy.\n- After successful source reads at 11:30:07Z and 11:30:30Z, generation continued without a completed mutation. The second write's argument stream began at 11:54:37Z and ended incomplete at 12:01:53Z after 4728 characters (advisory-generation-evidence.json; toolUse.totalToolCalls: 3; timing.timeToFirstArtifactMs: null).\n- The standard no-progress watchdog ended the trial after 1877 seconds without hard progress, at 44m23s elapsed (outcome.reason; outcome.durationMs: 2663183). Fixed metrics contain 1968 output tokens from completed requests; the unfinished request separately reached 5147 at about 2.74 tokens/sec. Persisted session content is incomplete after this mid-turn cutoff (perf.derived; log.txt; recording/chat-events.jsonl).\n\n## Tactical experiment — not applied\n\nFor a later A/B, set tuning.reasoning.thinkingBudget and tuning.profiles.thinking-general.reasoning.thinkingBudget from 4096 to 1024 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json, preserving output caps and deadlines. Test whether less post-retrieval reasoning permits the first grounded write to finish, on a tool-sourced brief and an unrelated incident report. This cannot fix the separate initial write-only policy defect and should be measured independently from that fix.\n\n## Strategic experiment — not applied\n\nIn packages/core/src/local-loop/constrained-turn.ts (isImmediateFileWritePrompt/isImmediateFileWriteTurn) and its consumption in packages/core/src/local-loop/llama-cpp-session.ts, suppress immediate write-only narrowing when the current task explicitly requires an unfulfilled lookup, including third-party MCP sources that have no workspace file path. Track successful retrieval evidence instead of interpreting “write the complete file now” as permission to remove research tools. Verify the tool surface retains the required lookup before any mutation, using this offline MCP pattern plus an unrelated connector-backed report. Preserve normal immediate writes when the required evidence is already supplied. Do not special-case source ids or scenario names.\n\n## Generalization and next experiment\n\nThe DocBlocks theme trial exposed a different evidence integration defect: read-only tool results were not registered for grounding. Here the first tool surface prevented retrieval before writing; after retrieval, the long unfinished write resembles the incident and spreadsheet failures. Keep policy ordering and slow-delivery experiments separate. A generic source-prerequisite A/B should reduce rejected unsourced first writes across at least two connector domains without harming compact tasks such as meeting-followup. No intervention or retry was applied to this completed failure.\n\n## Performance and fit\n\nRecorded decode 2.9 tokens/sec; prefill 161.38. Peak VRAM 10624/12227 MiB, process RSS 10322 MiB, whole-system RAM 28489/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Launch retained native 0.1.57/CUDA, 65536 context, one slot, f16 KV and 25 CPU FFN layers, with dense FFN of 39/64 layers on GPU. No explicit CUDA OOM, context overflow, unexpected native exit or transport error found in this trial; native shutdown followed the verdict normally. CPU/host-RAM placement is intentional; shared GPU memory and paging are not separately measured. \n\n## Last three same-scenario trials\n\n- [wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/facts.json): pass, fixed 9.5, 186.3 seconds, recorded decode 106.88 tokens/sec.\n- [wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc/facts.json): pass, fixed 9.5, 103.9 seconds, recorded decode 111.98 tokens/sec.\n- [wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-12-19-390Z-6729/facts.json): pass, fixed 9.5, 234.0 seconds, recorded decode 116.89 tokens/sec.\n\nAll three older Gemma4 12B/native 0.1.54 trials passed. Model, native release and source identity differ, so the current failure is historical contrast, not a controlled regression attributable to 0.1.57.\n" }, { "cellId": "r1-saluki-cuda-productivity-craftbook-week-plan", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-week-plan", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T12:04:36.262Z", "finishedAt": "2026-11-04T13:37:28.780Z", "durationMs": 5572518, "success": true, "reason": "craftbook-week-plan passed 12 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1198938 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-week-plan", "bytes": 3521, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.9917017654103016 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1198938 }, { "source": "facts", "path": "outcome.durationMs", "value": 5572518 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" }, "facts": { "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 937, "peakRssMb": 14708, "peakCpuPercent": 1212.3, "firstAtMs": 1, "lastAtMs": 5555984 }, "gpu": { "available": true, "sampleCount": 937, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 937, "peakUsedMb": 31306, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 90744, "totalOutputTokens": 12503, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 74104, "todayTokensCached": 65633, "todayTokensOut": 10421, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 74104, "totalTokensCached": 65633, "totalTokensOut": 10421, "totalCost": 0, "medianOutputTokensPerSec": 2.6, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.6, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7122, "cachedInputTokens": 6869, "outputTokens": 105, "cost": 0, "durationMs": 192942, "outputTokensPerSec": 2.8281176679689786, "at": "2026-11-04T13:34:40.149Z" }, "lastUpdated": "2026-11-04T13:34:40.149Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T13:37:20.570Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5557852, "meanTokensPerSec": 2.64, "genTokensPerSec": 2.64, "genTokens": 12503, "promptTokensPerSec": 151.21, "peakRssMb": 14708, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 74104, "output": 10421 } } } }, "craftbook": { "craftbookId": "week-plan", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "calendar-desk-week-plan-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T12:04:44.351Z", "completedAt": "2026-11-04T12:24:35.280Z", "ms": 1190929, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "6530a302-c2bc-46be-a114-fde7b7db92b1" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "plan", "activatedAt": "2026-11-04T12:24:35.282Z", "completedAt": "2026-11-04T12:51:39.890Z", "ms": 1624608, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "6530a302-c2bc-46be-a114-fde7b7db92b1" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "report", "activatedAt": "2026-11-04T12:51:39.891Z", "completedAt": "2026-11-04T13:01:12.632Z", "ms": 572741, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "e337f0be-7a7b-4b9d-ac17-20a391b7afa8" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T13:01:12.633Z", "completedAt": "2026-11-04T13:30:20.513Z", "ms": 1747880, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "cbdb6fc0-dcba-436c-a76b-c7d92c9d9b97" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T13:30:20.515Z", "completedAt": "2026-11-04T13:34:40.100Z", "ms": 259585, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "c3d29445-018d-4fa5-a637-7af4a645374e" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T12:04:42.191Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T12:04:42.087Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1190929 }, "sessions": { "total": 7, "taskScoped": 4, "byGezel": { "ambrose": 1, "eleanor": 1, "lech": 1, "meike": 2, "samuel": 1, "vojtech": 1 }, "perTask": { "calendar-desk-week-plan-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.230499267578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.230499267578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/week-plan", "knowledge://bendyline/handboek/craftbook/weekly-menu-plan", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/plan", "knowledge://bendyline/handboek/craftbook/onboarding-checklist" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-week-plan passed 12 deterministic craftbook checks", "durationMs": 5572518, "timeoutMs": 5619147, "budgetUsedFraction": 0.9917017654103016 }, "timing": { "startedAt": "2026-11-04T12:04:36.262Z", "finishedAt": "2026-11-04T13:37:28.780Z", "timeToFirstArtifactMs": 1198938, "timeToLastArtifactWriteMs": 5563018, "timeToFirstTokenMs": 110951, "firstTurnTtftMs": 96136, "timeToFirstToolCallMs": 133977 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 19, "byTool": { "list_artifacts": 2, "read_artifact": 4, "read_artifacts": 3, "read_file": 3, "write_artifact": 4, "write_task_note": 1, "advance_task_step": 1, "copy_artifact_to_workspace": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 16 }, "sniff": { "progression": [ { "atMs": 8272, "filePath": "craftbook-week-plan", "bytes": 0, "score": 0, "scoreMax": 12, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: followups.json" }, { "atMs": 351498, "filePath": "craftbook-week-plan", "bytes": 0, "score": 1, "scoreMax": 12, "signals": [], "failReason": "task sourced from craftbook week-plan has not finished: its gates have not accepted the result yet" }, { "atMs": 5408739, "filePath": "craftbook-week-plan", "bytes": 0, "score": 2, "scoreMax": 12, "signals": [], "failReason": "tasks/eval/report.md is 0 bytes, need ≥ 1200" }, { "atMs": 5563835, "filePath": "craftbook-week-plan", "bytes": 3521, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-week-plan", "bytes": 3521, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T12:04:36.266Z] [trial] id=craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi scenario=craftbook-week-plan model=underdog-saluki-27b-iq2-mix", "[2026-11-04T12:04:36.266Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T12:04:36.266Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-week-plan-FU8Vge", "[2026-11-04T12:04:36.266Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "[2026-11-04T12:04:36.266Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T12:04:36.269Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T12:04:36.273Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T12:04:36.280Z] [trial] throughput-scaled ceiling: 45m → 94m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T12:04:36.280Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-week-plan-FU8Vge", "[2026-11-04T12:04:42.116Z] [trial] daemon spawned pid=18744 port=https://127.0.0.1:62788", "[2026-11-04T12:04:43.712Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T12:04:43.727Z] [trial] meester=ambrose", "[2026-11-04T12:04:43.728Z] [trial] running scenario setup hook", "[2026-11-04T12:04:44.449Z] [trial] scenario setup complete", "[2026-11-04T12:04:44.450Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T12:04:44.494Z] [poll] initial digests hard=e606cf8378fab09a soft=27d2b84733c90e51", "[2026-11-04T12:04:44.583Z] [poll] hard-progress: e606cf8378fab09a → 7711a6e9125f1f3a after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T12:04:49.947Z] [poll] hard-progress: 7711a6e9125f1f3a → af42a117c6e7c788 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T12:06:54.994Z] [poll] hard-progress: af42a117c6e7c788 → 645dc1e8adf16291 after 125s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=44,stream=40) sniff=0/0B)", "[2026-11-04T12:10:27.806Z] [poll] hard-progress: 645dc1e8adf16291 → 8b49a726f905dc9e after 213s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=523,stream=970) sniff=1/0B)", "[2026-11-04T12:24:41.231Z] [poll] hard-progress: 8b49a726f905dc9e → 4340547de50b79f6 after 853s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=2591,stream=4246) sniff=1/0B)", "[2026-11-04T12:28:34.159Z] [poll] hard-progress: 4340547de50b79f6 → 876d598b5d5e02f7 after 233s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=6,slot=2836,stream=4491) sniff=1/0B)", "[2026-11-04T12:51:45.828Z] [poll] hard-progress: 876d598b5d5e02f7 → e4ebbed5f8eca42a after 1392s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=4,tools=7,slot=6264,stream=9765) sniff=1/0B)", "[2026-11-04T12:53:03.041Z] [poll] hard-progress: e4ebbed5f8eca42a → 75b0f6c2ea09e249 after 77s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=4,tools=9,slot=6294,stream=9808) sniff=1/0B)", "[2026-11-04T12:53:33.018Z] [poll] hard-progress: 75b0f6c2ea09e249 → 3f9f21174e79e3b6 after 30s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=4,tools=10,slot=6307,stream=9824) sniff=1/0B)" ] }, "metrics": { "process": { "sampleCount": 937, "peakRssMb": 14708, "peakCpuPercent": 1212.3, "firstAtMs": 1, "lastAtMs": 5555984 }, "gpu": { "available": true, "sampleCount": 937, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 937, "peakUsedMb": 31306, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 90744, "totalOutputTokens": 12503, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 74104, "todayTokensCached": 65633, "todayTokensOut": 10421, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 74104, "totalTokensCached": 65633, "totalTokensOut": 10421, "totalCost": 0, "medianOutputTokensPerSec": 2.6, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.6, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7122, "cachedInputTokens": 6869, "outputTokens": 105, "cost": 0, "durationMs": 192942, "outputTokensPerSec": 2.8281176679689786, "at": "2026-11-04T13:34:40.149Z" }, "lastUpdated": "2026-11-04T13:34:40.149Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T13:37:20.570Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5557852, "meanTokensPerSec": 2.64, "genTokensPerSec": 2.64, "genTokens": 12503, "promptTokensPerSec": 151.21, "peakRssMb": 14708, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 74104, "output": 10421 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T12:04:36.262Z", "finishedAt": "2026-11-04T13:37:28.780Z", "timeToFirstArtifactMs": 1198938, "timeToLastArtifactWriteMs": 5563018, "timeToFirstTokenMs": 110951, "firstTurnTtftMs": 96136, "timeToFirstToolCallMs": 133977 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "calendar-desk-week-plan-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T12:04:44.351Z", "completedAt": "2026-11-04T12:24:35.280Z", "ms": 1190929, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "6530a302-c2bc-46be-a114-fde7b7db92b1" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "plan", "activatedAt": "2026-11-04T12:24:35.282Z", "completedAt": "2026-11-04T12:51:39.890Z", "ms": 1624608, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "6530a302-c2bc-46be-a114-fde7b7db92b1" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "report", "activatedAt": "2026-11-04T12:51:39.891Z", "completedAt": "2026-11-04T13:01:12.632Z", "ms": 572741, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "e337f0be-7a7b-4b9d-ac17-20a391b7afa8" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T13:01:12.633Z", "completedAt": "2026-11-04T13:30:20.513Z", "ms": 1747880, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "cbdb6fc0-dcba-436c-a76b-c7d92c9d9b97" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T13:30:20.515Z", "completedAt": "2026-11-04T13:34:40.100Z", "ms": 259585, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "c3d29445-018d-4fa5-a637-7af4a645374e" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T12:04:42.191Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T12:04:42.087Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1190929 }, "sessions": { "total": 7, "taskScoped": 4, "byGezel": { "ambrose": 1, "eleanor": 1, "lech": 1, "meike": 2, "samuel": 1, "vojtech": 1 }, "perTask": { "calendar-desk-week-plan-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.230499267578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.230499267578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-craftbook-week-plan", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "scenarioId": "craftbook-week-plan", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-04T14:20:19.310Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "artifactBytes": 3651, "artifactSha256": "2c51abc9dab8c0d42b1cc16a33fc94ea9e4752342d5a8d3f6f5d1bbfdacd2d56", "artifactKind": "markdown", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/we/week-plan/versions/1.0.5/test.json", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ], "scoreAxes": { "coverage": 7, "prioritization": 4, "conflict-handling": 6, "realism": 2 }, "meanScore": 4.75, "justification": "All eight meetings and three follow-ups are covered, with a concrete fix for the Tuesday overlap. Yet Tuesday is called full despite five hours of slack, Friday is wrongly called lightest, and the largest focus block has no named work. The proposed reschedule reads as already moved in the daily plan.", "reviewBasis": "Re-read final captured report, verified both final copies match the provisional hash, and inspected the workflow reviewer verdict and final copy tool. Prior review read all eight calendar events and three follow-ups. Fixed pass is unchanged; calendar-realism concerns remain advisory.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md", "status": "present", "bytes": 3651 } ], "finalizationNote": "Standard runner feedback after workflow completion led to copy_artifact_to_workspace at 13:37:20Z. The previously missing workspace output now exists. No manual copying or gate changes." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:13.589Z", "reviewId": "021", "cellId": "r1-saluki-cuda-productivity-craftbook-week-plan", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "scenarioId": "craftbook-week-plan", "status": "scored", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-week-plan", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "4945e584eef46e7187579430c76371a63f6e23228d79f4162121c1440f72c628", "evidenceSha256": "6046f4dbdd7a83cc4a73c17f8ac5131159608734e25a6ad3747f7c79f710ffc0", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "bytes": 3651, "sha256": "2c51abc9dab8c0d42b1cc16a33fc94ea9e4752342d5a8d3f6f5d1bbfdacd2d56", "shownBytes": 3651, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md", "bytes": 3651, "sha256": "2c51abc9dab8c0d42b1cc16a33fc94ea9e4752342d5a8d3f6f5d1bbfdacd2d56", "shownBytes": 3651, "truncated": false } ], "contextFiles": [], "scoreAxes": { "coverage": 7, "prioritization": 5, "conflict-handling": 8, "realism": 5 }, "justification": "All 8 meetings, 4 prep flags and 3 follow-ups appear; no obvious inventions. The Tue overlap is named with a one-decision fix (move 1:1 to 11:30) plus a fallback. But Thursday's 6.5h block names no work, FU-11 sits on the \"tight\" Tuesday, and Tuesday is called tight while 12–17 is free.", "limitations": "The source calendar wasn't supplied, so coverage and invention can't be checked against it. Judged from the final report only. The artifact is report.md for a scope step; the scope and plan artifacts weren't shown.", "meanScore": 6.25, "durationMs": 18424, "costUsd": 0.03634, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6784, "cache_read_input_tokens": 0, "output_tokens": 920, "output_tokens_details": { "thinking_tokens": 574 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6784, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 920, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 6784, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6784 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 920, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 6784, "webSearchRequests": 0, "costUSD": 0.03634, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 574, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6784, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-week-plan / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 92m 53s; budget used: 99.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1198938`; `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":3521,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.9917017654103016`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 1198938`; `facts.outcome.durationMs: 5572518` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 5 + 0.15 * 10 = **9.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14708 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31306 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.64 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 90744 + 12503 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1191s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 23% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1198938`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":3521,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.9917017654103016`\n- `facts.toolUse.totalToolCalls: 19`\n- `facts.timing.timeToFirstArtifactMs: 1198938`\n- `facts.outcome.durationMs: 5572518`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1198938`; `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":3521,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.9917017654103016`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 1198938`; `facts.outcome.durationMs: 5572518`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Final advisory inline judge\n\nThe final 3651-byte report exists in both the artifacts drawer and expected workspace location. Re-read it and verified both copies match the provisional SHA-256; the content did not change. The primary declared axes from week-plan 1.0.5 test.json therefore retain these advisory scores, separate from fixed composite 9.0:\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| coverage | 7 | Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented. |\n| prioritization | 4 | Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room. |\n| conflict-handling | 6 | The overlapping pair is named with a concrete proposed fix the owner can accept in one decision. |\n| realism | 2 | The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy. |\n\n**Advisory mean: 4.75/10, codex-inline/current-session.**\n\nAll eight meetings and three follow-ups are covered, with a concrete fix for the Tuesday overlap. Yet Tuesday is called full despite five hours of slack, Friday is wrongly called lightest, and the largest focus block has no named work. The proposed reschedule reads as already moved in the daily plan.\n\nTuesday's own schedule leaves 12:00–17:00 free, contradicting “no room to absorb it.” Friday has two hours of meetings, while Wednesday and Thursday have one each, so calling Friday lightest is unsupported. A 6.5-hour Thursday focus block is not tied to a named priority. The conflict section proposes a move, but the daily schedule labels the meeting “moved” before approval. All original events remain accounted for; this review does not claim a calendar was actually changed.\n\n## Positives\n\n- The complete craftbook ran through scope, plan, report, evaluate and finish; all 12 fixed checks passed (craftbook.reachedTerminal: true; continuity.steps.perStep; sniff.latest.score: 12/12). This is demonstrated multi-phase completion, not only a generated plan fragment.\n- The report includes all eight events, all three follow-ups, a prep queue and the real Tuesday overlap. It was delivered to the requested workspace via one successful copy (toolUse.byTool.copy_artifact_to_workspace: 1; outcome.success; advisory-finalization-evidence.json). No fixed-rubric red flag, missing role or user intervention occurred (toolUse.redFlags; team.missingExpectedRoles; autoAnswer.total).\n\n## Negatives\n\n- Completion took 5,572,518 ms, 92m52s, using 99.2% of its scaled budget; first artifact took almost 20 minutes (outcome.durationMs; outcome.budgetUsedFraction; timing.timeToFirstArtifactMs: 1198938). Nineteen calls and eight created gezels produced a relatively short report (toolUse.totalToolCalls; team.totalGezelsCreated). Fixed efficiency and decode speed answer different questions.\n- Evaluation alone took 1,747,880 ms, about 29m8s, and approved the inconsistent load picture without revising the report (continuity.steps.perStep; toolUse.byTool.write_artifact: 4; artifacts/tasks/eval/verdict.md). The reviewer explicitly marked “load picture is honest” PASS. This is an advisory quality gap remaining behind a fixed pass.\n- The report existed in artifacts from 13:01:12Z, but the eval's workspace checks stayed empty until 13:37:20Z (timing.timeToLastArtifactWriteMs; sniff.progression; recording/chat-events.jsonl). Most of that interval was workflow evaluation/finish; the post-terminal destination repair took about 2m35s. Do not attribute the entire interval to the path mismatch.\n\n## Output-location discrepancy and normal recovery\n\nThe resolved craftbook report prompt explicitly says to use write_artifact/read_artifact and leave the workspace untouched; its runtime gates have artifact:true. The test.json success deliverable omits artifact:true, so the eval expects the same relative path in the workspace. These contracts conflicted, as preserved in advisory-progress-review.json.\n\nAfter the workflow reached complete at 13:34:40Z, standard missing-deliverable feedback reached the planner. At 13:37:20Z the model successfully called copy_artifact_to_workspace, making both copies available and satisfying all fixed checks. This was the runner's normal continuation, not a manual repair or capability retry. The earlier provisional noArtifact applies only to the workspace at that observation time; final output is present. Preserve the successful outcome and the discrepancy independently.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/pl/planner/versions/1.1.0/about.md, add: “Calculate occupied time and the largest free interval before describing each day's load. Keep proposed moves visibly pending, and name the work assigned to a long block or say that its priority is still unassigned.” Test on two unrelated calendars, measuring correct load rankings, proposal status and actionable block labels. The report's own five-hour gap and the unchanged reviewer PASS motivate this generic consistency instruction; do not embed fixture-specific weekdays or values.\n\n## Strategic experiments — not applied\n\nPrototype a reusable calendar-capacity calculation in a proposed packages/service/src/tasks/calendar-capacity.ts: union event intervals, derive actual gaps, and distinguish original bookings from proposed changes before generating prose. Compare its quantities against the report's load claims, with no calendar mutation. Validate overlapping, all-day and timezone cases in a separate change; numerical consistency should support rather than replace the user's scheduling judgment.\n\nFor a future catalog version, align the week-plan test.json deliverable surface with the craftbook's explicit artifact contract, or explicitly add an intended export step if workspace delivery is the product contract. Add contract validation in evals/src/craftbooks/scenario.ts that detects conflicting surfaces before launching a campaign. This corrects a real specification discrepancy, not an invitation to relax checks for this result; it changes comparison identity and must not be applied mid-campaign.\n\n## Performance and fit\n\nRecorded decode 2.64 tokens/sec; prefill 151.21. Peak VRAM 10642/12227 MiB, process RSS 14708 MiB, whole-system RAM 31306/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57/CUDA used one launch, 65536 context, one slot, f16 KV and 25 CPU FFN layers, with dense FFN of 39/64 layers on GPU. No explicit OOM, context overflow, unexpected native exit or transport error found. Whole-system RAM approached capacity but includes other processes; paging and shared GPU memory are not separately measured.\n\n## Generalization and next experiment\n\nUnlike the unfinished spreadsheet and PowerPoint workflows, this longer workflow reached terminal completion, albeit close to its deadline. Compact plan-and-estimate entries passed in roughly nine minutes; the week plan's longer preparation and review did not guarantee consistent scheduling advice. A separate generic capacity-check experiment should improve realism across unrelated calendars, while a later reasoning-budget A/B can measure latency without conflating it with planning quality. No current tuning, prompts, gates or output files were changed.\n\n## Last three same-scenario trials\n\n- [craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/facts.json): pass, fixed 8.5, 372.4 seconds, recorded decode 117.06 tokens/sec.\n- [craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/facts.json): pass, fixed 8.5, 517.4 seconds, recorded decode 126.33 tokens/sec.\n- [craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-21-09-815Z-cjud/facts.json): pass, fixed 8.5, 495.7 seconds, recorded decode 121.16 tokens/sec.\n\nAll three older Gemma4 12B/native 0.1.54 trials passed the 12 checks. This is historical context only: different model, source and native identities prevent a controlled speed or quality improvement claim from the current 9.0 versus 8.5 scores.\n" }, { "cellId": "r1-saluki-cuda-productivity-craftbook-ab-test-readout", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-ab-test-readout", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T13:40:24.958Z", "finishedAt": "2026-11-04T15:00:53.223Z", "durationMs": 4828265, "success": true, "reason": "craftbook-ab-test-readout passed 19 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 431052 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-ab-test-readout", "bytes": 470, "score": 19, "scoreMax": 19, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.8592523028851177 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 431052 }, { "source": "facts", "path": "outcome.durationMs", "value": 4828265 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" }, "facts": { "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 813, "peakRssMb": 14131, "peakCpuPercent": 1252, "firstAtMs": 1, "lastAtMs": 4813142 }, "gpu": { "available": true, "sampleCount": 813, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 813, "peakUsedMb": 31153, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 59622, "totalOutputTokens": 11900, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 60764, "todayTokensCached": 59241, "todayTokensOut": 3784, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 60764, "totalTokensCached": 59241, "totalTokensOut": 3784, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8771, "cachedInputTokens": 8410, "outputTokens": 491, "cost": 0, "durationMs": 665899, "outputTokensPerSec": 3.9158337721531202, "at": "2026-11-04T15:00:40.527Z" }, "lastUpdated": "2026-11-04T15:00:40.527Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T15:00:45.995Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4814602, "meanTokensPerSec": 2.74, "genTokensPerSec": 2.74, "genTokens": 11900, "promptTokensPerSec": 158.98, "peakRssMb": 14131, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 60764, "output": 3784 } } } }, "craftbook": { "craftbookId": "ab-test-readout", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "a-b-test-readout-eval/1", "stepId": "hypothesis-scope", "activatedAt": "2026-11-04T13:40:32.809Z", "completedAt": "2026-11-04T13:47:36.085Z", "ms": 423276, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "ff944c7d-db68-4fca-b917-82f134543b22" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "analyze", "activatedAt": "2026-11-04T13:47:36.086Z", "completedAt": "2026-11-04T14:19:15.772Z", "ms": 1899686, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "ff944c7d-db68-4fca-b917-82f134543b22" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "write", "activatedAt": "2026-11-04T14:19:15.773Z", "completedAt": "2026-11-04T14:34:16.488Z", "ms": 900715, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "afc0fb37-17bc-4196-b377-8d0ec1b14c9b" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T14:34:16.490Z", "completedAt": "2026-11-04T14:49:32.273Z", "ms": 915783, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "c678a488-8fe9-4452-9ef5-a0b7d9e891d9" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T14:49:32.274Z", "completedAt": "2026-11-04T15:00:40.455Z", "ms": 668181, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "d683be1e-8a01-4385-96f4-fdd5925f5c1b" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T13:40:30.876Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T13:40:30.756Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 900715 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "ani": 1, "malai": 1, "noor": 1, "roxana": 1 }, "perTask": { "a-b-test-readout-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.205413818359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.205413818359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/ab-test-readout", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/how-we-test-models", "knowledge://bendyline/handboek/craftbook/cohort-analysis", "knowledge://bendyline/handboek/whats-new/1.26273" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-ab-test-readout passed 19 deterministic craftbook checks", "durationMs": 4828265, "timeoutMs": 5619147, "budgetUsedFraction": 0.8592523028851177 }, "timing": { "startedAt": "2026-11-04T13:40:24.958Z", "finishedAt": "2026-11-04T15:00:53.223Z", "timeToFirstArtifactMs": 431052, "timeToLastArtifactWriteMs": 4147136, "timeToFirstTokenMs": 101974, "firstTurnTtftMs": 87037, "timeToFirstToolCallMs": 137402 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Data Analyst", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 19, "byTool": { "read_file": 6, "write_artifact": 4, "write_file": 1, "write_task_note": 2, "advance_task_step": 2, "read_artifact": 4 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 8102, "filePath": "craftbook-ab-test-readout", "bytes": 0, "score": 0, "scoreMax": 19, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/experiment-brief.md, data/checkout-test.csv" }, { "atMs": 138709, "filePath": "craftbook-ab-test-readout", "bytes": 0, "score": 1, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" }, { "atMs": 2187405, "filePath": "craftbook-ab-test-readout", "bytes": 470, "score": 11, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" }, { "atMs": 3231513, "filePath": "craftbook-ab-test-readout", "bytes": 470, "score": 18, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" }, { "atMs": 4820511, "filePath": "craftbook-ab-test-readout", "bytes": 470, "score": 19, "scoreMax": 19, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-ab-test-readout", "bytes": 470, "score": 19, "scoreMax": 19, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T13:40:24.962Z] [trial] id=craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi scenario=craftbook-ab-test-readout model=underdog-saluki-27b-iq2-mix", "[2026-11-04T13:40:24.962Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "[2026-11-04T13:40:24.962Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-ab-test-readout-RDdHPg", "[2026-11-04T13:40:24.963Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T13:40:24.963Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T13:40:24.966Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T13:40:24.969Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T13:40:24.975Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-ab-test-readout-RDdHPg", "[2026-11-04T13:40:24.975Z] [trial] throughput-scaled ceiling: 45m → 94m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T13:40:30.814Z] [trial] daemon spawned pid=23292 port=https://127.0.0.1:61598", "[2026-11-04T13:40:32.312Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T13:40:32.328Z] [trial] meester=ambrose", "[2026-11-04T13:40:32.328Z] [trial] running scenario setup hook", "[2026-11-04T13:40:32.896Z] [trial] scenario setup complete", "[2026-11-04T13:40:32.897Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T13:40:32.947Z] [poll] initial digests hard=406ade4bda7ee84a soft=7012d13c7ccbee7a", "[2026-11-04T13:40:33.105Z] [poll] hard-progress: 406ade4bda7ee84a → 52086640f65aa355 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T13:40:38.317Z] [poll] hard-progress: 52086640f65aa355 → bfb86d14b5b61a7a after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T13:42:43.712Z] [poll] hard-progress: bfb86d14b5b61a7a → d5dae1f94d4bcd0b after 125s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=67,stream=76) sniff=1/0B)", "[2026-11-04T13:47:39.394Z] [poll] hard-progress: d5dae1f94d4bcd0b → ad15a0dc3db12881 after 296s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=3,slot=791,stream=1372) sniff=1/0B)", "[2026-11-04T14:16:52.423Z] [poll] hard-progress: ad15a0dc3db12881 → 645068f70f337e3e after 1753s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=4,slot=4985,stream=5783) sniff=11/470B)", "[2026-11-04T14:18:51.123Z] [poll] hard-progress: 645068f70f337e3e → bd2db6ebbc2f076b after 119s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=5,slot=5242,stream=6265) sniff=11/470B)", "[2026-11-04T14:19:20.641Z] [poll] hard-progress: bd2db6ebbc2f076b → a283493e29949f4f after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=5269,stream=6308) sniff=11/470B)", "[2026-11-04T14:20:45.306Z] [poll] hard-progress: a283493e29949f4f → 7f5fb09a24b8ef29 after 85s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=3,tools=8,slot=5299,stream=6351) sniff=11/470B)", "[2026-11-04T14:21:03.162Z] [poll] hard-progress: 7f5fb09a24b8ef29 → c194d899de55412e after 18s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=3,tools=9,slot=5313,stream=6373) sniff=11/470B)" ] }, "metrics": { "process": { "sampleCount": 813, "peakRssMb": 14131, "peakCpuPercent": 1252, "firstAtMs": 1, "lastAtMs": 4813142 }, "gpu": { "available": true, "sampleCount": 813, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 813, "peakUsedMb": 31153, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 59622, "totalOutputTokens": 11900, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 60764, "todayTokensCached": 59241, "todayTokensOut": 3784, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 60764, "totalTokensCached": 59241, "totalTokensOut": 3784, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8771, "cachedInputTokens": 8410, "outputTokens": 491, "cost": 0, "durationMs": 665899, "outputTokensPerSec": 3.9158337721531202, "at": "2026-11-04T15:00:40.527Z" }, "lastUpdated": "2026-11-04T15:00:40.527Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T15:00:45.995Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4814602, "meanTokensPerSec": 2.74, "genTokensPerSec": 2.74, "genTokens": 11900, "promptTokensPerSec": 158.98, "peakRssMb": 14131, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 60764, "output": 3784 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T13:40:24.958Z", "finishedAt": "2026-11-04T15:00:53.223Z", "timeToFirstArtifactMs": 431052, "timeToLastArtifactWriteMs": 4147136, "timeToFirstTokenMs": 101974, "firstTurnTtftMs": 87037, "timeToFirstToolCallMs": 137402 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "a-b-test-readout-eval/1", "stepId": "hypothesis-scope", "activatedAt": "2026-11-04T13:40:32.809Z", "completedAt": "2026-11-04T13:47:36.085Z", "ms": 423276, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "ff944c7d-db68-4fca-b917-82f134543b22" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "analyze", "activatedAt": "2026-11-04T13:47:36.086Z", "completedAt": "2026-11-04T14:19:15.772Z", "ms": 1899686, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "ff944c7d-db68-4fca-b917-82f134543b22" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "write", "activatedAt": "2026-11-04T14:19:15.773Z", "completedAt": "2026-11-04T14:34:16.488Z", "ms": 900715, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "afc0fb37-17bc-4196-b377-8d0ec1b14c9b" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T14:34:16.490Z", "completedAt": "2026-11-04T14:49:32.273Z", "ms": 915783, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "c678a488-8fe9-4452-9ef5-a0b7d9e891d9" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T14:49:32.274Z", "completedAt": "2026-11-04T15:00:40.455Z", "ms": 668181, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "d683be1e-8a01-4385-96f4-fdd5925f5c1b" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T13:40:30.876Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T13:40:30.756Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 900715 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "ani": 1, "malai": 1, "noor": 1, "roxana": 1 }, "perTask": { "a-b-test-readout-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.205413818359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.205413818359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "scenarioId": "craftbook-ab-test-readout", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/artifacts/a-b-test-readout-eval/tasks/eval/readout.md", "artifactBytes": 3749, "artifactKind": "markdown", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/ab/ab-test-readout/versions/1.0.6/test.json", "axes": [ { "name": "decision rule", "description": "The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift." }, { "name": "statistical reporting", "description": "Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json." }, { "name": "honest caveats", "description": "Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action." } ], "scoreAxes": { "decision rule": 8, "statistical reporting": 8, "honest caveats": 4 }, "meanScore": 6.67, "justification": "The do-not-ship decision correctly honors the breached refund guardrail; all stated rates, lifts, CI and p-value match the statistics. Caveats overstate “real, not noise” and “small but real,” offer no power calculation, and call for a tighter guardrail while retaining the same threshold. Refund-reason investigation is actionable.", "reviewBasis": "Full 3749-byte readout, hypothesis scope, reviewer verdict, source brief, raw two-arm CSV and emitted stats.json read. Rates, lifts, normal-approximation CI and p-value independently recomputed. No gates or artifacts changed. Model identity visible in paths." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:13.793Z", "reviewId": "022", "cellId": "r1-saluki-cuda-productivity-craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "scenarioId": "craftbook-ab-test-readout", "status": "scored", "axes": [ { "name": "decision rule", "description": "The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift." }, { "name": "statistical reporting", "description": "Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json." }, { "name": "honest caveats", "description": "Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-ab-test-readout", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "81ac839dcf7832a05d51497a5a652c26e5d87865149f0cba4c69741e7869fa28", "evidenceSha256": "d417dec1f02676bb6dc7aac5b1bb04370cf546067c229297248d51aaf9bc1192", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/artifacts/a-b-test-readout-eval/tasks/eval/readout.md", "bytes": 3749, "sha256": "e7110b60850807f4b678ae1ed7e61eeeffacdc40b1b3f8123594aa6ad1e36f44", "shownBytes": 3749, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi\\workspace\\a-b-test-readout-eval\\data\\checkout-test.csv", "bytes": 95, "sha256": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "shownBytes": 95, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi\\workspace\\a-b-test-readout-eval\\data\\stats.json", "bytes": 470, "sha256": "4de153609018af8d546b4d505836aafe96a5463092b21791afa8ac64e42f5774", "shownBytes": 470, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi\\workspace\\a-b-test-readout-eval\\source\\experiment-brief.md", "bytes": 504, "sha256": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "shownBytes": 504, "truncated": false } ], "scoreAxes": { "decision rule": 9, "statistical reporting": 9, "honest caveats": 6 }, "justification": "The readout quotes the three-clause rule and says do not ship because the +0.23 pp refund delta exceeds the 0.20 pp limit, despite p=0.0079. All figures match stats.json and the CSV: 12.00% vs 13.25%, +1.25 pp, +10.42% relative, CI [0.33, 2.17]. Units are right. Caveats are partly generic, and the refund signal is overstated.", "limitations": "The artifact is the final readout; the earlier hypothesis-scope output was not shown. I recomputed the CI and p-value by hand and they agree. The refund delta's own significance is untested, and the report calls it \"real\", which is not supported. No power calculation is given.", "meanScore": 8, "durationMs": 18352, "costUsd": 0.032106, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5868, "cache_read_input_tokens": 0, "output_tokens": 863, "output_tokens_details": { "thinking_tokens": 493 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5868, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 863, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 5868, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5868 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 863, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 5868, "webSearchRequests": 0, "costUSD": 0.032106, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 493, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5868, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-ab-test-readout / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 80m 28s; budget used: 85.9%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 431052`; `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":470,\"score\":19,\"scoreMax\":19,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.8592523028851177`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 431052`; `facts.outcome.durationMs: 4828265` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 5 + 0.15 * 10 = **9.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14131 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31153 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.74 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 59622 + 11900 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 901s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 21% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 431052`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":470,\"score\":19,\"scoreMax\":19,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.8592523028851177`\n- `facts.toolUse.totalToolCalls: 19`\n- `facts.timing.timeToFirstArtifactMs: 431052`\n- `facts.outcome.durationMs: 4828265`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 431052`; `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":470,\"score\":19,\"scoreMax\":19,\"signals\":[],\"failReason\":null}`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.8592523028851177`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 431052`; `facts.outcome.durationMs: 4828265`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nRead the complete 3749-byte declared readout plus source brief, CSV, stats.json, locked hypothesis and reviewer verdict. The exact axes from ab-test-readout 1.0.6 test.json are judged independently of fixed composite 9.0:\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| decision rule | 8 | The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift. |\n| statistical reporting | 8 | Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json. |\n| honest caveats | 4 | Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action. |\n\n**Advisory mean: 6.67/10, codex-inline/current-session.**\n\nThe do-not-ship decision correctly honors the breached refund guardrail; all stated rates, lifts, CI and p-value match the statistics. Caveats overstate “real, not noise” and “small but real,” offer no power calculation, and call for a tighter guardrail while retaining the same threshold. Refund-reason investigation is actionable.\n\nIndependent arithmetic confirms conversion rates 12.00% and 13.25%, +1.25 pp absolute lift, +10.4167% relative lift and a normal-approximation 95% interval about [0.33, 2.17] pp. The pooled two-sided p-value is about 0.007785 versus the artifact's 0.0079, a small numerical discrepancy that does not affect the significance decision. Refund rates are 0.80% and 1.03%; their +0.23 pp observed delta exceeds the predeclared 0.20 pp threshold. The requested decision rule therefore yields do_not_ship. Detailed calculations are in advisory-arithmetic-check.json, separate from fixed facts.\n\nThe readout overstates inference with “real, not noise,” “not a fluke” and “small but real.” Crossing the refund threshold is a decision-rule result, not proof of a causal refund mechanism or statistical significance of that secondary rate. A separate simple refund-difference interval is approximately [-0.034, 0.494] pp, including zero; this does not change the stipulated rule. The “solid sample” caveat does not calculate power or state a minimum detectable effect. Next actions usefully request refund reasons, but their heading asks for a tighter guardrail while the body keeps 0.20 pp.\n\n## Positives\n\n- All 19 deterministic checks passed; the actual craftbook reached its terminal step (sniff.latest.score: 19/19; craftbook.reachedTerminal: true; continuity.steps.perStep). The recommendation honors the guardrail despite a statistically positive primary result.\n- Machine-readable statistics and a human-readable artifact agree; no fixed-rubric red flag, missing role or user intervention was recorded (toolUse.byTool.write_file: 1; write_artifact: 4; toolUse.redFlags; team.missingExpectedRoles; autoAnswer.total). The independent arithmetic check supports the recorded rates and decision.\n\n## Negatives\n\n- Elapsed time was 4,828,265 ms, 80m28s, using 85.9% of its budget, with 19 calls and eight created gezels (outcome.durationMs; outcome.budgetUsedFraction; toolUse.totalToolCalls; team.totalGezelsCreated). Analysis occupied about 31m40s, review 15m16s, and finishing alone 11m8s (continuity.steps.perStep). The fixed score does not express this latency.\n- A correct derived margin, 0.23 - 0.20 = 0.03 pp, caused the first readout write to be refused because that number was absent from the evidence text. The model reread stats.json and re-emitted the full document without the margin, costing roughly seven minutes between writes (toolUse.byTool.write_artifact: 4; recording/chat-events.jsonl at 14:27:26Z and 14:34:16Z; advisory-grounding-refusal.json). This is a derived-evidence handling problem, not incorrect arithmetic.\n- The reviewer approved the narrative while leaving the certainty and power caveats above unchanged (continuity.steps.perStep evaluate completed; artifacts/tasks/eval/verdict.md; toolUse.byTool). Correct values and a correct decision do not make every explanation statistically calibrated.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/co/copywriter/versions/1.1.0/about.md, add: “Separate an observed threshold breach from evidence of a causal or population effect. Use uncertainty language rather than ‘proven’ or ‘not noise’; if power was not calculated, say so. Keep follow-up thresholds consistent across headings and instructions.” Test on an unrelated retention experiment and an observational metrics summary, checking numerical fidelity, calibrated claims and actionable recommendations. Do not bake in this fixture's percentages or decision.\n\n## Strategic experiments — not applied\n\nPrototype a generic two-proportion calculator in a proposed packages/script-stdlib/src/proportion-comparison.ts that returns counts, rates, difference intervals, test method and derived threshold margin through the normal script boundary. Register its derived outputs as evidence through packages/service/src/chat/evidence-ledger.ts, retaining source counts and expression provenance. This can avoid long mental calculation and refusal of a correct computed 0.03 pp without allowing arbitrary invented figures. Validate two unrelated metrics tasks and incorrect-input cases in a separate change.\n\nMeasure per-phase latency before changing orchestration. A later isolated A/B can cap this model's general reasoning allowance from 4096 to 1024 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json while preserving output limits, phase gates and sampling. Compare readout, calendar planning and coding review; require unchanged factual accuracy and earlier completion before treating it as an improvement. No current tuning or gates were changed.\n\n## Performance and fit\n\nRecorded decode 2.74 tokens/sec; prefill 158.98. Peak VRAM 10640/12227 MiB, process RSS 14131 MiB, whole-system RAM 31153/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57/CUDA retained one launch, 65536 context, one slot, f16 KV and 25 CPU FFN layers, with dense FFN of 39/64 layers on GPU. No explicit OOM, context overflow, unexpected native exit or transport error was found. Recorded context fill peaked at about 20.5%, with zero compactions (continuity.compaction). Shared GPU memory and paging are not separately sampled; high whole-system RAM includes other processes.\n\n## Generalization and next experiment\n\nThe readout and week-plan both completed multi-phase workflows, unlike spreadsheet-model and PowerPoint. This readout's arithmetic is sound, while the spreadsheet scope contained an incorrect subtraction; that contrast motivates deterministic numeric support rather than assuming every numerical task fails. The same campaign also exposed false missing-evidence refusals for DocBlocks outputs and proposed slide content. Test derived-evidence registration across numeric and document tasks separately from any reasoning-budget change, retaining the frozen baseline.\n\n## Last three same-scenario trials\n\n- [craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/facts.json): pass, fixed 8.5, 287.9 seconds, recorded decode 124.5 tokens/sec.\n- [craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/facts.json): fail, fixed 1.2, 4778.0 seconds, recorded decode 116.94 tokens/sec.\n- [craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-44-21-123Z-np4s/facts.json): pass, fixed 9.5, 291.6 seconds, recorded decode 121.64 tokens/sec.\n\nThe older Gemma4 12B/native 0.1.54 sequence includes two passes and one chat-stall failure. The failed attempt reported a reachable daemon issuing no model turns; this trial remained active and completed. Different model, source and native identities make these historical context, not a controlled speed or reliability regression.\n" }, { "cellId": "r1-saluki-cuda-productivity-craftbook-annotated-bibliography", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-annotated-bibliography", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T15:04:12.087Z", "finishedAt": "2026-11-04T15:25:25.284Z", "durationMs": 1273197, "success": true, "reason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 116504 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-annotated-bibliography", "bytes": 6251, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2265818993523394 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 18 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 116504 }, { "source": "facts", "path": "outcome.durationMs", "value": 1273197 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 212, "peakRssMb": 14268, "peakCpuPercent": 1201.6, "firstAtMs": 1, "lastAtMs": 1260366 }, "gpu": { "available": true, "sampleCount": 212, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 212, "peakUsedMb": 31031, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 50702, "totalOutputTokens": 9107, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 50893, "todayTokensCached": 48762, "todayTokensOut": 6997, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 50893, "totalTokensCached": 48762, "totalTokensOut": 6997, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7174, "cachedInputTokens": 6894, "outputTokens": 77, "cost": 0, "durationMs": 60263, "outputTokensPerSec": 9.328904395055533, "at": "2026-11-04T15:25:16.976Z" }, "lastUpdated": "2026-11-04T15:25:16.976Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T15:25:20.395Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1261777, "meanTokensPerSec": 8.96, "genTokensPerSec": 8.96, "genTokens": 9107, "promptTokensPerSec": 441.17, "peakRssMb": 14268, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 50893, "output": 6997 } } } }, "craftbook": { "craftbookId": "annotated-bibliography", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "annotated-bibliography-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T15:04:20.006Z", "completedAt": "2026-11-04T15:06:08.694Z", "ms": 108688, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "5848d96b-51ed-4a14-a7d5-03a06a5db007" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "research", "activatedAt": "2026-11-04T15:06:08.696Z", "completedAt": "2026-11-04T15:17:10.598Z", "ms": 661902, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "5801c2e2-4899-404e-b229-03a1ccc035de" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "write", "activatedAt": "2026-11-04T15:17:10.599Z", "completedAt": "2026-11-04T15:20:47.805Z", "ms": 217206, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "b263bf1c-812b-4121-8a7f-78edee964de8" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T15:20:47.806Z", "completedAt": "2026-11-04T15:23:13.556Z", "ms": 145750, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "36cfc417-c3da-4ee5-b1e8-207986d04e2a" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T15:23:13.557Z", "completedAt": "2026-11-04T15:25:16.932Z", "ms": 123375, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "258217b8-cbc5-459e-af99-865e6970e97a" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T15:04:18.188Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T15:04:18.080Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 145750 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "bente": 1, "ishaan": 1, "kenji": 1, "pyotr": 1, "raffaele": 1, "rustam": 1, "thabo": 1 }, "perTask": { "annotated-bibliography-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.15985107421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.15985107421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/annotated-bibliography", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/reading-list", "knowledge://bendyline/handboek/gezel-app-page-style-guide", "knowledge://bendyline/handboek/craftbook/cohort-analysis" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "durationMs": 1273197, "timeoutMs": 5619147, "budgetUsedFraction": 0.2265818993523394 }, "timing": { "startedAt": "2026-11-04T15:04:12.087Z", "finishedAt": "2026-11-04T15:25:25.284Z", "timeToFirstArtifactMs": 116504, "timeToLastArtifactWriteMs": 1141299, "timeToFirstTokenMs": 35820, "firstTurnTtftMs": 21337, "timeToFirstToolCallMs": 41758 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 18, "byTool": { "read_file": 5, "write_artifact": 3, "read_artifact": 5, "write_file": 1, "write_task_note": 2, "advance_task_step": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 8171, "filePath": "craftbook-annotated-bibliography", "bytes": 0, "score": 0, "scoreMax": 9, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/source-cards.md" }, { "atMs": 42942, "filePath": "craftbook-annotated-bibliography", "bytes": 0, "score": 1, "scoreMax": 9, "signals": [], "failReason": "task sourced from craftbook annotated-bibliography has not finished: its gates have not accepted the result yet" }, { "atMs": 975546, "filePath": "craftbook-annotated-bibliography", "bytes": 6251, "score": 8, "scoreMax": 9, "signals": [], "failReason": "task sourced from craftbook annotated-bibliography has not finished: its gates have not accepted the result yet" }, { "atMs": 1267788, "filePath": "craftbook-annotated-bibliography", "bytes": 6251, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-annotated-bibliography", "bytes": 6251, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T15:04:12.090Z] [trial] id=craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo scenario=craftbook-annotated-bibliography model=underdog-saluki-27b-iq2-mix", "[2026-11-04T15:04:12.090Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-annotated-bibliography-UxZfio", "[2026-11-04T15:04:12.090Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T15:04:12.090Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T15:04:12.090Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "[2026-11-04T15:04:12.093Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T15:04:12.096Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T15:04:12.102Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-annotated-bibliography-UxZfio", "[2026-11-04T15:04:12.102Z] [trial] throughput-scaled ceiling: 45m → 94m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T15:04:18.128Z] [trial] daemon spawned pid=30616 port=https://127.0.0.1:60095", "[2026-11-04T15:04:19.502Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T15:04:19.520Z] [trial] meester=kenji", "[2026-11-04T15:04:19.521Z] [trial] running scenario setup hook", "[2026-11-04T15:04:20.161Z] [trial] scenario setup complete", "[2026-11-04T15:04:20.163Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T15:04:20.214Z] [poll] initial digests hard=670ec85e8a3fec03 soft=157c2b5415d51f4b", "[2026-11-04T15:04:20.298Z] [poll] hard-progress: 670ec85e8a3fec03 → a0dda1d2d7aa52ab after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T15:04:25.432Z] [poll] hard-progress: a0dda1d2d7aa52ab → c3f1f474fccd7b98 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T15:04:55.071Z] [poll] hard-progress: c3f1f474fccd7b98 → 725e96efaef750c8 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=24,stream=43) sniff=1/0B)", "[2026-11-04T15:06:13.332Z] [poll] hard-progress: 725e96efaef750c8 → b2ed58268b93e8d1 after 78s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=237,stream=1132) sniff=1/0B)", "[2026-11-04T15:06:19.613Z] [poll] hard-progress: b2ed58268b93e8d1 → f56c80061e43409f after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=238,stream=1132) sniff=1/0B)", "[2026-11-04T15:06:55.413Z] [poll] hard-progress: f56c80061e43409f → 33a40f468f59058c after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=271,stream=1210) sniff=1/0B)", "[2026-11-04T15:17:10.642Z] [poll] hard-progress: 33a40f468f59058c → 4742d35ab087f29b after 615s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=2083,stream=7978) sniff=1/0B)", "[2026-11-04T15:17:16.468Z] [poll] hard-progress: 4742d35ab087f29b → 8cddb17c84ab2492 after 6s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=5,slot=2083,stream=7978) sniff=1/0B)", "[2026-11-04T15:17:46.249Z] [poll] hard-progress: 8cddb17c84ab2492 → 8a2b7f9726081131 after 30s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=7,slot=2103,stream=8018) sniff=1/0B)" ] }, "metrics": { "process": { "sampleCount": 212, "peakRssMb": 14268, "peakCpuPercent": 1201.6, "firstAtMs": 1, "lastAtMs": 1260366 }, "gpu": { "available": true, "sampleCount": 212, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 212, "peakUsedMb": 31031, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 50702, "totalOutputTokens": 9107, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 50893, "todayTokensCached": 48762, "todayTokensOut": 6997, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 50893, "totalTokensCached": 48762, "totalTokensOut": 6997, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7174, "cachedInputTokens": 6894, "outputTokens": 77, "cost": 0, "durationMs": 60263, "outputTokensPerSec": 9.328904395055533, "at": "2026-11-04T15:25:16.976Z" }, "lastUpdated": "2026-11-04T15:25:16.976Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T15:25:20.395Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1261777, "meanTokensPerSec": 8.96, "genTokensPerSec": 8.96, "genTokens": 9107, "promptTokensPerSec": 441.17, "peakRssMb": 14268, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 50893, "output": 6997 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T15:04:12.087Z", "finishedAt": "2026-11-04T15:25:25.284Z", "timeToFirstArtifactMs": 116504, "timeToLastArtifactWriteMs": 1141299, "timeToFirstTokenMs": 35820, "firstTurnTtftMs": 21337, "timeToFirstToolCallMs": 41758 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "annotated-bibliography-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T15:04:20.006Z", "completedAt": "2026-11-04T15:06:08.694Z", "ms": 108688, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "5848d96b-51ed-4a14-a7d5-03a06a5db007" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "research", "activatedAt": "2026-11-04T15:06:08.696Z", "completedAt": "2026-11-04T15:17:10.598Z", "ms": 661902, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "5801c2e2-4899-404e-b229-03a1ccc035de" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "write", "activatedAt": "2026-11-04T15:17:10.599Z", "completedAt": "2026-11-04T15:20:47.805Z", "ms": 217206, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "b263bf1c-812b-4121-8a7f-78edee964de8" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T15:20:47.806Z", "completedAt": "2026-11-04T15:23:13.556Z", "ms": 145750, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "36cfc417-c3da-4ee5-b1e8-207986d04e2a" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T15:23:13.557Z", "completedAt": "2026-11-04T15:25:16.932Z", "ms": 123375, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "258217b8-cbc5-459e-af99-865e6970e97a" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T15:04:18.188Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T15:04:18.080Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 145750 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "bente": 1, "ishaan": 1, "kenji": 1, "pyotr": 1, "raffaele": 1, "rustam": 1, "thabo": 1 }, "perTask": { "annotated-bibliography-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.15985107421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.15985107421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "scenarioId": "craftbook-annotated-bibliography", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/workspace/annotated-bibliography-eval/bibliography.md", "artifactBytes": 6269, "artifactSha256": "f8b32bfb911d130c0f3dd93581cc753070de5b01b14241971e9378ec0af1f1b7", "artifactKind": "markdown", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/an/annotated-bibliography/versions/1.0.4/test.json", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ], "scoreAxes": { "accuracy": 8, "grounding": 4, "structure": 8, "actionability": 5 }, "meanScore": 6.25, "justification": "All six citations and supplied figures are preserved in clear, consistent annotations. Grounding slips: it asserts uncontrolled confounders, adherence-only outcomes, specific connectivity failures and a phone-plus-video hybrid that the cards do not specify. Relevance is clear, but next actions remain general.", "reviewBasis": "Read the full bibliography, all six supplied offline source cards and the workflow reviewer verdict. Final trace contains only one bibliography write; the same output was captured at termination. Scores assess fidelity to the provided fixtures, not external-study truth or clinical recommendations. Model identity visible in paths." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:23.069Z", "reviewId": "023", "cellId": "r1-saluki-cuda-productivity-craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "scenarioId": "craftbook-annotated-bibliography", "status": "scored", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-annotated-bibliography", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "ed71c9eea7c26ab5fa684ea2aaed8f39e0dc53aa54e29115227f01bf2829e393", "evidenceSha256": "bbdb858c77d880909558ec85bfdf3d9fcefa62f63a6bb63d7099a9fa1d5bb3b2", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/workspace/annotated-bibliography-eval/bibliography.md", "bytes": 6269, "sha256": "f8b32bfb911d130c0f3dd93581cc753070de5b01b14241971e9378ec0af1f1b7", "shownBytes": 6269, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo\\workspace\\annotated-bibliography-eval\\source\\source-cards.md", "bytes": 1294, "sha256": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "shownBytes": 1294, "truncated": false } ], "scoreAxes": { "accuracy": 8, "grounding": 6, "structure": 9, "actionability": 6 }, "justification": "All figures match the cards (11%, 72h, 900, n=240, 18 pts, 42, 1,600). Initials are correct and the order is alphabetical. Some details are invented: Chen's \"dead zones, slow speeds, outages\", Allen's uncontrolled \"comorbidities\", and Williams' \"framework\". Structure is uniform and clean. Relevance notes are useful but there are no synthesis or next steps.", "limitations": "Only bibliography.md was shown. The scope.md and sources.md artifacts and the step process were not visible. The axes are generic and fit a bibliography loosely. The actionability score reflects the relevance annotations alone.", "meanScore": 7.25, "durationMs": 18186, "costUsd": 0.034224, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6265, "cache_read_input_tokens": 0, "output_tokens": 916, "output_tokens_details": { "thinking_tokens": 559 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6265, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 916, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 6265, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6265 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 916, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 6265, "webSearchRequests": 0, "costUSD": 0.034224, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 559, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6265, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-annotated-bibliography / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 21m 13s; budget used: 22.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 116504`; `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":6251,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.2265818993523394`; `facts.toolUse.totalToolCalls: 18`; `facts.timing.timeToFirstArtifactMs: 116504`; `facts.outcome.durationMs: 1273197` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14268 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31031 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.96 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 50702 + 9107 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 146s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.71 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 16% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 116504`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":6251,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.2265818993523394`\n- `facts.toolUse.totalToolCalls: 18`\n- `facts.timing.timeToFirstArtifactMs: 116504`\n- `facts.outcome.durationMs: 1273197`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 116504`; `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":6251,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.2265818993523394`; `facts.toolUse.totalToolCalls: 18`; `facts.timing.timeToFirstArtifactMs: 116504`; `facts.outcome.durationMs: 1273197`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nRead the complete 6269-byte bibliography, all six source cards and the reviewer verdict. Fixed composite remains 9.5. The exact declared axes from annotated-bibliography 1.0.4 test.json are scored separately:\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| accuracy | 8 | Computed figures are correct against the seeded records. |\n| grounding | 4 | Every claim traces to the seeded data — nothing invented. |\n| structure | 8 | Tables and sections make the analysis checkable at a glance. |\n| actionability | 5 | Recommendations follow from the findings and are concretely doable. |\n\n**Advisory mean: 6.25/10, codex-inline/current-session.**\n\nAll six citations and supplied figures are preserved in clear, consistent annotations. Grounding slips: it asserts uncontrolled confounders, adherence-only outcomes, specific connectivity failures and a phone-plus-video hybrid that the cards do not specify. Relevance is clear, but next actions remain general.\n\nThe six citations preserve names, years, titles and journals without inventing volume/page/DOI fields absent from the cards. All reported source numbers match. The grounding concerns concern added claims: observational does not mean the study controlled no confounders; the adherence card does not establish that no other outcomes were measured; the cost card does not define the hybrid as phone plus video only; and specific connectivity failure modes were not supplied. These plausible additions should be framed as possibilities or missing information, not findings. This evaluation compares the output to offline fixtures and does not verify real clinical literature.\n\n## Positives\n\n- All nine fixed checks passed and the actual craftbook reached terminal completion (sniff.latest.score: 9/9; craftbook.reachedTerminal: true). The final file contains six consistently formatted entries in alphabetical order, each with Summary, Critical evaluation and Relevance.\n- One completed workspace write produced the bibliography, with no fixed-rubric red flag, missing role or user intervention (toolUse.byTool.write_file: 1; toolUse.redFlags; team.missingExpectedRoles; autoAnswer.total). First artifact appeared at 116504 ms and the whole task completed in 21m13s (timing.timeToFirstArtifactMs; outcome.durationMs: 1273197).\n\n## Negatives\n\n- The reviewer marked the annotations and citation data as non-fabricated without correcting unsupported method and modality claims (toolUse.byTool.write_artifact: 3; artifacts/tasks/1/verdict.md; advisory-artifact-review.json). The fixed gates and verdict establish structured coverage, not sentence-level support for every elaboration.\n- Eighteen calls and nine created gezels were used for a six-entry document (toolUse.totalToolCalls; team.totalGezelsCreated). The main source packet took over ten minutes after its input reads; preserving basic source facts still required substantial generation (recording/chat-events.jsonl at 15:06:49Z and 15:17:10Z; perf.derived.genTokens: 9107).\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/re/researcher/versions/1.2.0/about.md, add: “Do not turn an omitted method detail into a claim that the study lacked it. Separate source-reported findings from possible limitations, and label missing controls, outcomes or intervention components as unspecified.” Apply equivalent review guidance to the bibliography authoring role in a later version if needed. Test on an unrelated policy evidence summary and a technical literature review, measuring unsupported assertions while retaining useful critical evaluation. Do not insert fixture names or expected facts.\n\n## Strategic experiment — not applied\n\nPrototype a sentence-level claim/source review in a proposed packages/service/src/chat/claim-review.ts, recording each assertion with its supporting excerpt and a status of source-reported, inference or unknown. Focus on unsupported negations and added specifics that simple name/number grounding cannot detect. Integrate through existing task/review boundaries; do not claim deterministic string matching proves entailment. Compare research summaries and software-documentation synthesis before adoption, and retain the current fixed results as baseline.\n\n## Performance and fit\n\nRecorded decode 8.96 tokens/sec; prefill 441.17. Peak VRAM 10640/12227 MiB, process RSS 14268 MiB, whole-system RAM 31031/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57/CUDA still used one launch, 65536 context, f16 KV, one slot and 25 CPU FFN layers, with 39/64 dense FFN layers on GPU. No explicit OOM/context overflow/unexpected native exit or transport error found. Shared GPU memory and paging are not separately measured.\n\nThis 8.96-token/sec mean is substantially faster than the preceding A/B readout's 2.74 despite the same recorded launch configuration. No model tuning or hardware settings were changed by this monitoring work. Workload and host conditions are not controlled well enough to identify the cause; preserve the observed variation rather than assuming a constant campaign throughput.\n\n## Generalization and next experiment\n\nThe A/B readout and week-plan also passed their fixed gates while reviewers missed certainty or consistency problems. This bibliography adds a third domain with plausible unsupported elaboration. A broad claim-source review experiment should target that shared behavior without teaching scenario-specific answers. The earlier Wikipedia research brief produced no artifact, whereas this task completed an existing-source synthesis; successful source-based prose and successful research-tool sequencing are distinct capabilities.\n\n## Last three same-scenario trials\n\n- [craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/facts.json): pass, fixed 9.5, 106.5 seconds, recorded decode 122.68 tokens/sec.\n- [craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts/facts.json): pass, fixed 9, 180.5 seconds, recorded decode 120.44 tokens/sec.\n- [craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-13-44-208Z-78gn/facts.json): pass, fixed 9, 167.8 seconds, recorded decode 122.2 tokens/sec.\n\nAll three older Gemma4 12B/native 0.1.54 trials passed. Their model, native and source identities differ; the comparison is historical context, not a controlled speed or quality regression.\n" }, { "cellId": "r1-saluki-cuda-productivity-craftbook-research-to-document", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-research-to-document", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T15:28:35.769Z", "finishedAt": "2026-11-04T15:48:13.795Z", "durationMs": 1178026, "success": true, "reason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 151916 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-research-to-document", "bytes": 1697, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.20964498704162748 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 39 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 151916 }, { "source": "facts", "path": "outcome.durationMs", "value": 1178026 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 196, "peakRssMb": 14594, "peakCpuPercent": 1333, "firstAtMs": 1, "lastAtMs": 1161479 }, "gpu": { "available": true, "sampleCount": 196, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 196, "peakUsedMb": 31406, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 98407, "totalOutputTokens": 7043, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 100470, "todayTokensCached": 98634, "todayTokensOut": 1651, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 100470, "totalTokensCached": 98634, "totalTokensOut": 1651, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10108, "cachedInputTokens": 10039, "outputTokens": 48, "cost": 0, "durationMs": 127025, "outputTokensPerSec": 8.87959365468016, "at": "2026-11-04T15:48:02.844Z" }, "lastUpdated": "2026-11-04T15:48:02.844Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T15:48:05.819Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1162887, "meanTokensPerSec": 9.01, "genTokensPerSec": 9.01, "genTokens": 7043, "promptTokensPerSec": 365.62, "peakRssMb": 14594, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 100470, "output": 1651 } } } }, "craftbook": { "craftbookId": "research-to-document", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "word-document-from-content-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T15:28:44.312Z", "completedAt": "2026-11-04T15:31:07.847Z", "ms": 143535, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "40503391-44d4-4457-98d7-77a1a10fe2a5" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "sources", "activatedAt": "2026-11-04T15:31:07.849Z", "completedAt": "2026-11-04T15:32:56.813Z", "ms": 108964, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "edd538f9-8024-4462-9275-c87aaadaa4cc" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "write", "activatedAt": "2026-11-04T15:32:56.815Z", "completedAt": "2026-11-04T15:34:49.018Z", "ms": 112203, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "213e73ad-e5bf-4a8b-a9ae-d4dbc0faa448" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "publish", "activatedAt": "2026-11-04T15:34:49.019Z", "completedAt": "2026-11-04T15:38:13.018Z", "ms": 203999, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "213e73ad-e5bf-4a8b-a9ae-d4dbc0faa448" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T15:38:13.020Z", "completedAt": "2026-11-04T15:45:53.323Z", "ms": 460303, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "4325a040-8dde-4798-b1bc-172a30867940" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T15:45:53.324Z", "completedAt": "2026-11-04T15:48:02.792Z", "ms": 129468, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "ec4633ef-d6be-4569-8b55-a8e956c1493b" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T15:28:42.502Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T15:28:42.411Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 136501.5 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "bilal": 1, "ngoc": 2, "oumou": 1, "rahul": 1 }, "perTask": { "word-document-from-content-eval/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.3127899169921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3127899169921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/project-type/support-desk", "knowledge://bendyline/handboek/craftbook/morning-brief", "knowledge://bendyline/handboek/craftbook/source-quality-audit", "knowledge://bendyline/handboek/project-type/hiring-desk", "knowledge://bendyline/handboek/project-type/vendor-contracts" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "durationMs": 1178026, "timeoutMs": 5619147, "budgetUsedFraction": 0.20964498704162748 }, "timing": { "startedAt": "2026-11-04T15:28:35.769Z", "finishedAt": "2026-11-04T15:48:13.795Z", "timeToFirstArtifactMs": 151916, "timeToLastArtifactWriteMs": 373294, "timeToFirstTokenMs": 34618, "firstTurnTtftMs": 20106, "timeToFirstToolCallMs": 41190 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Planner", "Researcher", "Copywriter", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 39, "byTool": { "read_file": 7, "list_dir": 1, "read_task_notes": 3, "write_artifact": 2, "read_artifact": 4, "write_file": 1, "write_task_note": 4, "advance_task_step": 4, "list_roots": 2, "convert_document": 1, "inspect_document": 2, "preview_document": 2, "save_artifact": 1, "stat": 2, "validate": 1, "list_artifacts": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [ { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153257_word-document-docblocks-artifact-surface/tool-0-img-0.png", "bytes": 19310 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153257_word-document-docblocks-artifact-surface/tool-0-img-1.png", "bytes": 113390 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153257_word-document-docblocks-artifact-surface/tool-0-img-2.png", "bytes": 38313 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153257_word-document-docblocks-artifact-surface/tool-0-img-3.png", "bytes": 98573 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153813_new-session/tool-0-img-0.png", "bytes": 19310 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153813_new-session/tool-0-img-1.png", "bytes": 113390 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153813_new-session/tool-0-img-2.png", "bytes": 38313 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153813_new-session/tool-0-img-3.png", "bytes": 98573 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-04_153813_new-session/tool-0-img-4.png", "bytes": 93574 } ], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 8835, "filePath": "craftbook-research-to-document", "bytes": 0, "score": 1, "scoreMax": 14, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/brief.md" }, { "atMs": 43454, "filePath": "craftbook-research-to-document", "bytes": 0, "score": 2, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 351830, "filePath": "craftbook-research-to-document", "bytes": 1697, "score": 8, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 465047, "filePath": "craftbook-research-to-document", "bytes": 1697, "score": 9, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 482602, "filePath": "craftbook-research-to-document", "bytes": 1697, "score": 10, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 506266, "filePath": "craftbook-research-to-document", "bytes": 1697, "score": 11, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 529918, "filePath": "craftbook-research-to-document", "bytes": 1697, "score": 13, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 1169549, "filePath": "craftbook-research-to-document", "bytes": 1697, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-research-to-document", "bytes": 1697, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T15:28:35.774Z] [trial] id=craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5 scenario=craftbook-research-to-document model=underdog-saluki-27b-iq2-mix", "[2026-11-04T15:28:35.774Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T15:28:35.774Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "[2026-11-04T15:28:35.774Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-research-to-document-o0lXzs", "[2026-11-04T15:28:35.776Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T15:28:35.774Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T15:28:35.780Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T15:28:35.787Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-research-to-document-o0lXzs", "[2026-11-04T15:28:35.787Z] [trial] throughput-scaled ceiling: 45m → 94m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T15:28:42.401Z] [trial] daemon spawned pid=41912 port=https://127.0.0.1:53968", "[2026-11-04T15:28:43.816Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T15:28:43.871Z] [trial] meester=zara", "[2026-11-04T15:28:43.871Z] [trial] running scenario setup hook", "[2026-11-04T15:28:44.466Z] [trial] scenario setup complete", "[2026-11-04T15:28:44.467Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T15:28:44.536Z] [poll] initial digests hard=40f25bf2bcb8592d soft=a4f0d55a8c97025d", "[2026-11-04T15:28:44.649Z] [poll] hard-progress: 40f25bf2bcb8592d → a296050cef071a8e after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T15:28:49.763Z] [poll] hard-progress: a296050cef071a8e → 6ad8419cdaf3fd86 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T15:29:19.290Z] [poll] hard-progress: 6ad8419cdaf3fd86 → 3bea292188f3ec87 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=29,stream=55) sniff=2/0B)", "[2026-11-04T15:29:31.166Z] [poll] hard-progress: 3bea292188f3ec87 → 79bbb778eda62d8e after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=51,stream=126) sniff=2/0B)", "[2026-11-04T15:31:11.750Z] [poll] hard-progress: 79bbb778eda62d8e → 420b021e556f6dd9 after 101s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=337,stream=1505) sniff=2/0B)", "[2026-11-04T15:31:17.878Z] [poll] hard-progress: 420b021e556f6dd9 → af0e021e8430d91c after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=338,stream=1505) sniff=2/0B)", "[2026-11-04T15:31:59.481Z] [poll] hard-progress: af0e021e8430d91c → e0ae01e8d9b31e8a after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=393,stream=1643) sniff=2/0B)", "[2026-11-04T15:32:58.432Z] [poll] hard-progress: e0ae01e8d9b31e8a → 47b3c30f558f6357 after 59s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=7,slot=554,stream=2484) sniff=2/0B)", "[2026-11-04T15:33:34.390Z] [poll] hard-progress: 47b3c30f558f6357 → 764511cf624f2989 after 36s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=9,slot=576,stream=2527) sniff=2/0B)" ] }, "metrics": { "process": { "sampleCount": 196, "peakRssMb": 14594, "peakCpuPercent": 1333, "firstAtMs": 1, "lastAtMs": 1161479 }, "gpu": { "available": true, "sampleCount": 196, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 196, "peakUsedMb": 31406, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 98407, "totalOutputTokens": 7043, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 100470, "todayTokensCached": 98634, "todayTokensOut": 1651, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 100470, "totalTokensCached": 98634, "totalTokensOut": 1651, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10108, "cachedInputTokens": 10039, "outputTokens": 48, "cost": 0, "durationMs": 127025, "outputTokensPerSec": 8.87959365468016, "at": "2026-11-04T15:48:02.844Z" }, "lastUpdated": "2026-11-04T15:48:02.844Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T15:48:05.819Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1162887, "meanTokensPerSec": 9.01, "genTokensPerSec": 9.01, "genTokens": 7043, "promptTokensPerSec": 365.62, "peakRssMb": 14594, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 100470, "output": 1651 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T15:28:35.769Z", "finishedAt": "2026-11-04T15:48:13.795Z", "timeToFirstArtifactMs": 151916, "timeToLastArtifactWriteMs": 373294, "timeToFirstTokenMs": 34618, "firstTurnTtftMs": 20106, "timeToFirstToolCallMs": 41190 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "word-document-from-content-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T15:28:44.312Z", "completedAt": "2026-11-04T15:31:07.847Z", "ms": 143535, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "40503391-44d4-4457-98d7-77a1a10fe2a5" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "sources", "activatedAt": "2026-11-04T15:31:07.849Z", "completedAt": "2026-11-04T15:32:56.813Z", "ms": 108964, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "edd538f9-8024-4462-9275-c87aaadaa4cc" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "write", "activatedAt": "2026-11-04T15:32:56.815Z", "completedAt": "2026-11-04T15:34:49.018Z", "ms": 112203, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "213e73ad-e5bf-4a8b-a9ae-d4dbc0faa448" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "publish", "activatedAt": "2026-11-04T15:34:49.019Z", "completedAt": "2026-11-04T15:38:13.018Z", "ms": 203999, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "213e73ad-e5bf-4a8b-a9ae-d4dbc0faa448" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T15:38:13.020Z", "completedAt": "2026-11-04T15:45:53.323Z", "ms": 460303, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "4325a040-8dde-4798-b1bc-172a30867940" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T15:45:53.324Z", "completedAt": "2026-11-04T15:48:02.792Z", "ms": 129468, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "ec4633ef-d6be-4569-8b55-a8e956c1493b" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T15:28:42.502Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T15:28:42.411Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 136501.5 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "bilal": 1, "ngoc": 2, "oumou": 1, "rahul": 1 }, "perTask": { "word-document-from-content-eval/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.3127899169921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3127899169921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-productivity-craftbook-research-to-document", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "scenarioId": "craftbook-research-to-document", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/workspace/word-document-from-content-eval/documents/eval/report.md", "artifactBytes": 1707, "artifactSha256": "1750b5d88eab54586917e151cf8d029a46bbfc6590dc6dbbfce25c32817dad1a", "artifacts": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/workspace/word-document-from-content-eval/documents/eval/report.md", "bytes": 1707, "sha256": "1750b5d88eab54586917e151cf8d029a46bbfc6590dc6dbbfce25c32817dad1a" } ], "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/re/research-to-document/versions/1.2.8/test.json", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ], "scoreAxes": { "grounding": 7, "structure": 7, "convertibility": 8 }, "meanScore": 7.33, "justification": "All supplied figures and three actions are retained in complete Markdown and an editable DOCX. The before/after coverage row assumes an unprovided baseline, and next actions are described as agreed and already addressing the complaint. Clear structure, with repetitive citations; native pagination remains unverified.", "reviewBasis": "Read the complete Markdown, brief, scope/source logs, task notes and converted DOCX XML. The final 18797-byte DOCX contains the source text and one table. Inspected the captured reconstructed preview images for summary, metrics and actions; these are structural previews, not Word-native pagination. Actual tool results identify DocBlocks 2.7.5 and real artifact URIs, despite mock definitions in the catalog test spec. Final DOCX hash is checked against the converter output below.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/artifacts/word-document-from-content-eval/tasks/eval/report.docx", "bytes": 18797, "sha256": "225653364c57aa5bbd6a709fed17810c51cfad31086b67e046fc0f5878403a23", "status": "present", "review": "ZIP document XML preserves source text and table; native pagination unverified." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:30.103Z", "reviewId": "024", "cellId": "r1-saluki-cuda-productivity-craftbook-research-to-document", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "scenarioId": "craftbook-research-to-document", "status": "scored", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-research-to-document", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "35744a102948054d4d2a148f3c5b94da994a28ff4969546d552ba1cc751083ed", "evidenceSha256": "075365c075381275a6ad65cd8defffa12a7953d800c81b1ef973604c43b1ce5c", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/workspace/word-document-from-content-eval/documents/eval/report.md", "bytes": 1707, "sha256": "1750b5d88eab54586917e151cf8d029a46bbfc6590dc6dbbfce25c32817dad1a", "shownBytes": 1707, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5\\workspace\\word-document-from-content-eval\\source\\brief.md", "bytes": 405, "sha256": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "shownBytes": 405, "truncated": false } ], "scoreAxes": { "grounding": 7, "structure": 8, "convertibility": 8 }, "justification": "All numbers (18 SKUs, 18→6 hours, 14.2%→8.9%) match the brief, and no new metrics appear. Small embellishments: \"agreed\" next actions, the claim that all three actions address the status-silence complaint, and invented rationales. The table lists 18 SKUs under both Before and After. The structure is complete, with clear headings, a table and numbered actions.", "limitations": "Source-only review of the Markdown. No DOCX conversion, diagnostics or preview was shown. The step was \"scope\", but the full report was delivered, so I did not check the scope or lock artifacts. Citation-bracket rendering in DocBlocks is unverified.", "meanScore": 7.666666666666667, "durationMs": 16485, "costUsd": 0.022723999999999998, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4055, "cache_read_input_tokens": 0, "output_tokens": 650, "output_tokens_details": { "thinking_tokens": 294 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4055, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 650, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 4055, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4055 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 650, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 4055, "webSearchRequests": 0, "costUSD": 0.022723999999999998, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 294, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4055, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-research-to-document / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 19m 38s; budget used: 21.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 151916`; `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1697,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.20964498704162748`; `facts.toolUse.totalToolCalls: 39`; `facts.timing.timeToFirstArtifactMs: 151916`; `facts.outcome.durationMs: 1178026` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14594 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31406 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.01 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 98407 + 7043 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 8 → 6 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 137s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.63 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 31% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 151916`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1697,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.20964498704162748`\n- `facts.toolUse.totalToolCalls: 39`\n- `facts.timing.timeToFirstArtifactMs: 151916`\n- `facts.outcome.durationMs: 1178026`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 151916`; `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1697,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.20964498704162748`; `facts.toolUse.totalToolCalls: 39`; `facts.timing.timeToFirstArtifactMs: 151916`; `facts.outcome.durationMs: 1178026`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nRead the complete Markdown, brief, scope/source logs, task notes and converted DOCX XML. The final 18797-byte DOCX contains the source text and one table. Inspected the captured reconstructed preview images for summary, metrics and actions; these are structural previews, not Word-native pagination. Actual tool results identify DocBlocks 2.7.5 and real artifact URIs, despite mock definitions in the catalog test spec. Final DOCX hash is checked against the converter output below.\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| grounding | 7 | Every metric and action comes from the seeded brief. |\n| structure | 7 | The source has a usable title, summary, sections, table, and next actions. |\n| convertibility | 8 | The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML. |\n\nAdvisory mean: **7.33/10** (codex-inline/current-session).\n\nAll supplied figures and three actions are retained in complete Markdown and an editable DOCX. The before/after coverage row assumes an unprovided baseline, and next actions are described as agreed and already addressing the complaint. Clear structure, with repetitive citations; native pagination remains unverified.\n\nFixed composite remains **8.5/10**; the advice below does not alter the fixed scores or campaign identity.\n\n## Positives\n\n- All fourteen checks passed and the workflow reached its terminal step; the required source document, convert/preview/save calls and binary document are present (outcome.success; sniff.latest.score: 14/14; toolUse.byTool.convert_document: 1; toolUse.byTool.save_artifact: 1).\n- All supplied numerical values and all three named next actions are preserved, with clear headings and a real metrics table (artifacts.otherFileCount; documents/eval/report.md; source/brief.md). The workflow reviewer correctly disclosed reconstructed-preview limits instead of claiming verified native pagination (recording/task-notes.json; toolUse.byTool.preview_document: 2).\n\n## Negatives\n\n- The table assigns 18 SKUs to both Before and After, although the brief only supplies pilot coverage; “agreed” and “is addressed” add decision/completion certainty not supplied by the brief (sniff.latest.bytes: 1697; reviewed source artifact). These do not invalidate the fixed gate but reduce grounding.\n- Thirty-nine calls and nine created gezels supported a short single-source report (toolUse.totalToolCalls; team.totalGezelsCreated). Evaluation and finish consumed roughly ten minutes after publishing; the reviewer also repeats validate’s 18963-byte claim although the captured file and converter result are 18797 bytes (recording/task-notes.json; advisory-tool-evidence.json).\n\n## Tactical experiment — not applied\n\nIn a future version of ../gilde/data/gezel-templates/co/copywriter/versions/1.1.0/about.md, add: “Represent a single supplied coverage figure once; do not manufacture a baseline/after pair. Keep proposed actions distinct from agreed or completed actions.” Measure unsupported temporal claims on operational reports and unrelated meeting summaries. This addresses document fidelity without inserting scenario facts.\n\n## Strategic experiment — not applied\n\nPrototype a structured conversion receipt in packages/service/src/chat/manager.ts that carries saved-file byte count, SHA-256 and preview basis into later review turns. Reconcile validation/metadata discrepancies against the saved file instead of accepting the newest prose result. Measure fewer repeated verification calls and accurate limitation disclosure across DOCX and PPTX workflows; retain useful source review.\n\n## Performance and fit\n\nDuration 19.63 minutes; recorded decode 9.01 tokens/sec, prefill 365.62 tokens/sec, 7043 generated tokens. Peak VRAM 10642/12227 MiB, process RSS 14594 MiB, whole-system RAM 31406/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57 CUDA; 1 launch(es), 65536 context, one slot, f16 KV, 4096 reasoning budget. Planner: 39/64 dense FFN layers on GPU and 25 on CPU. This is mixed CPU/GPU residency, not full GPU residency. Shared GPU memory and paging are not separately measured. No explicit OOM, context overflow or unexpected native/transport incident was found in this trial.\n\n## Generalization and next experiment\n\nUnlike the earlier powerpoint-deck trial, this workflow completed conversion and persisted a real editable binary. Like annotated-bibliography and the A/B readout, fixed completeness coexists with small unsupported elaborations. Future source-fidelity guidance should be tested across those domains, without altering this campaign.\n\nAll suggestions are prospective; no prompts, gates, tuning, task artifacts or model files were edited. Retain this frozen campaign as the baseline and evaluate any later change on at least one unrelated domain before calling it a general improvement.\n\n## Last three same-scenario trials\n\n- [craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/facts.json): pass, fixed 8.5, 153.5 seconds, recorded decode 125.4 tokens/sec. craftbook-research-to-document passed 14 deterministic craftbook checks\n- [craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8/facts.json): pass, fixed 8.5, 272.3 seconds, recorded decode 119.82 tokens/sec. craftbook-research-to-document passed 14 deterministic craftbook checks\n- [craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-21-24-620Z-dxw0/facts.json): pass, fixed 8.5, 173.2 seconds, recorded decode 122.51 tokens/sec. craftbook-research-to-document passed 14 deterministic craftbook checks\n\nThese are historical comparisons, not a controlled native-version or model experiment: source/model/native identities may differ. Earlier outcomes: 3/3 passed.\n" }, { "cellId": "r1-saluki-cuda-developer-fix-squisq-bugs", "baselineCellId": "r1-saluki-cuda-developer-fix-squisq-bugs", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T15:51:11.511Z", "finishedAt": "2026-11-04T16:05:34.439Z", "durationMs": 862928, "success": true, "reason": "all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 167385 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "fix-squisq-bugs", "bytes": 7790, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2764246185150115 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 14 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 167385 }, { "source": "facts", "path": "outcome.durationMs", "value": 862928 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 143, "peakRssMb": 12957, "peakCpuPercent": 1207.4, "firstAtMs": 0, "lastAtMs": 844794 }, "gpu": { "available": true, "sampleCount": 143, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 143, "peakUsedMb": 31534, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 45183, "totalOutputTokens": 6054, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 30620, "todayTokensCached": 13716, "todayTokensOut": 1524, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 30620, "totalTokensCached": 13716, "totalTokensOut": 1524, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 16773, "outputTokens": 495, "cost": 0, "durationMs": 147052, "outputTokensPerSec": 8.82600327805623, "at": "2026-11-04T16:05:23.119Z" }, "lastUpdated": "2026-11-04T16:05:23.119Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T16:05:24.757Z" } }, "source": "engine-log" }, "derived": { "durationMs": 847024, "meanTokensPerSec": 8.66, "genTokensPerSec": 8.66, "genTokens": 6054, "promptTokensPerSec": 396.17, "peakRssMb": 12957, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 30620, "output": 1524 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T15:51:17.376Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T15:51:17.254Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "dani": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 7, "maxContextFill": 0.217742919921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.217742919921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)", "durationMs": 862928, "timeoutMs": 3121748, "budgetUsedFraction": 0.2764246185150115 }, "timing": { "startedAt": "2026-11-04T15:51:11.511Z", "finishedAt": "2026-11-04T16:05:34.439Z", "timeToFirstArtifactMs": 167385, "timeToLastArtifactWriteMs": 851585, "timeToFirstTokenMs": 25602, "firstTurnTtftMs": 13927, "timeToFirstToolCallMs": 37487 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 14, "byTool": { "read_files": 2, "replace_in_file": 6, "list_dir": 3, "read_file": 2, "run_installed_script": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 7801, "filePath": "fix-squisq-bugs", "bytes": 5461, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng 87.8 — it must cross the antimeridian (±180), not run back through 0" }, { "atMs": 172753, "filePath": "fix-squisq-bugs", "bytes": 5483, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng 87.8 — it must cross the antimeridian (±180), not run back through 0" }, { "atMs": 344661, "filePath": "fix-squisq-bugs", "bytes": 6591, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng -88.2 — it must cross the antimeridian (±180), not run back through 0" }, { "atMs": 380351, "filePath": "fix-squisq-bugs", "bytes": 6471, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng -88.2 — it must cross the antimeridian (±180), not run back through 0" }, { "atMs": 422259, "filePath": "fix-squisq-bugs", "bytes": 6401, "score": 2, "scoreMax": 5, "signals": [ "graceful-errors", "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng -88.2 — it must cross the antimeridian (±180), not run back through 0" }, { "atMs": 524000, "filePath": "fix-squisq-bugs", "bytes": 7006, "score": 3, "scoreMax": 5, "signals": [ "graceful-errors", "deep-layer-validation", "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng -88.2 — it must cross the antimeridian (±180), not run back through 0" }, { "atMs": 852827, "filePath": "fix-squisq-bugs", "bytes": 7790, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } ], "latest": { "filePath": "fix-squisq-bugs", "bytes": 7790, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T15:51:11.515Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "[2026-11-04T15:51:11.515Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-fix-squisq-bugs-76PcQC", "[2026-11-04T15:51:11.515Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T15:51:11.515Z] [trial] id=fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q scenario=fix-squisq-bugs model=underdog-saluki-27b-iq2-mix", "[2026-11-04T15:51:11.515Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T15:51:11.517Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T15:51:11.521Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T15:51:11.526Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-fix-squisq-bugs-76PcQC", "[2026-11-04T15:51:11.526Z] [trial] throughput-scaled ceiling: 25m → 52m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T15:51:17.331Z] [trial] daemon spawned pid=37732 port=https://127.0.0.1:59987", "[2026-11-04T15:51:18.582Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T15:51:18.599Z] [trial] meester=zephyr", "[2026-11-04T15:51:18.599Z] [trial] running scenario setup hook", "[2026-11-04T15:51:19.020Z] [trial] scenario setup complete", "[2026-11-04T15:51:19.022Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T15:51:19.091Z] [poll] initial digests hard=3ca0996941e2ba49 soft=5343c93455458c15", "[2026-11-04T15:51:19.459Z] [poll] hard-progress: 3ca0996941e2ba49 → b80e4cefcefb9a5a after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/5461B)", "[2026-11-04T15:51:54.151Z] [poll] hard-progress: b80e4cefcefb9a5a → df9b3af307de59ea after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=33,stream=125) sniff=1/5461B)", "[2026-11-04T15:54:04.337Z] [poll] hard-progress: df9b3af307de59ea → c204afb8f0a01e9a after 130s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=382,stream=1178) sniff=1/5483B)", "[2026-11-04T15:56:56.322Z] [poll] hard-progress: c204afb8f0a01e9a → 58dc2e3a1e1c7d84 after 172s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=850,stream=3396) sniff=1/6591B)", "[2026-11-04T15:57:31.905Z] [poll] hard-progress: 58dc2e3a1e1c7d84 → b07a517635193b19 after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=940,stream=3856) sniff=1/6471B)", "[2026-11-04T15:58:13.917Z] [poll] hard-progress: b07a517635193b19 → 5b0e8429aa907de5 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=1043,stream=4364) sniff=2/6401B)", "[2026-11-04T15:59:55.624Z] [poll] hard-progress: 5b0e8429aa907de5 → 3e0cf143e8450e92 after 102s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=1320,stream=5864) sniff=3/7006B)", "[2026-11-04T16:00:06.972Z] [poll] hard-progress: 3e0cf143e8450e92 → 810f1384a034931d after 11s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=1339,stream=5947) sniff=3/7006B)", "[2026-11-04T16:00:24.545Z] [poll] hard-progress: 810f1384a034931d → 62f44f6450bfe4cd after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=1364,stream=6010) sniff=3/7006B)" ] }, "metrics": { "process": { "sampleCount": 143, "peakRssMb": 12957, "peakCpuPercent": 1207.4, "firstAtMs": 0, "lastAtMs": 844794 }, "gpu": { "available": true, "sampleCount": 143, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 143, "peakUsedMb": 31534, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 45183, "totalOutputTokens": 6054, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 30620, "todayTokensCached": 13716, "todayTokensOut": 1524, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 30620, "totalTokensCached": 13716, "totalTokensOut": 1524, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 16773, "outputTokens": 495, "cost": 0, "durationMs": 147052, "outputTokensPerSec": 8.82600327805623, "at": "2026-11-04T16:05:23.119Z" }, "lastUpdated": "2026-11-04T16:05:23.119Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T16:05:24.757Z" } }, "source": "engine-log" }, "derived": { "durationMs": 847024, "meanTokensPerSec": 8.66, "genTokensPerSec": 8.66, "genTokens": 6054, "promptTokensPerSec": 396.17, "peakRssMb": 12957, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 30620, "output": 1524 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T15:51:11.511Z", "finishedAt": "2026-11-04T16:05:34.439Z", "timeToFirstArtifactMs": 167385, "timeToLastArtifactWriteMs": 851585, "timeToFirstTokenMs": 25602, "firstTurnTtftMs": 13927, "timeToFirstToolCallMs": 37487 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T15:51:17.376Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T15:51:17.254Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "dani": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 7, "maxContextFill": 0.217742919921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.217742919921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-fix-squisq-bugs", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "scenarioId": "fix-squisq-bugs", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "artifactBytes": 7792, "artifactSha256": "2a0696ebc7b8c08fa9f0faf52cde78c3d7a54f7caf37ab6aa7f5f60dba8eb41e", "artifacts": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "bytes": 5556, "sha256": "2a0696ebc7b8c08fa9f0faf52cde78c3d7a54f7caf37ab6aa7f5f60dba8eb41e" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/workspace/squisq-core/packages/core/src/imageEdit/persistence.ts", "bytes": 2236, "sha256": "65678bec0468a9249a3b5aa680a1ecfa1be56312e3f82bc8d46fdacd4b1aeaf9" } ], "axisSource": null, "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared qualitative axes. The five fixed checks pass, but independent endpoint probes show that the saved spherical interpolation is still incorrect. Graceful empty returns and per-layer field validation are present. A native transport failure recovered automatically before completion.", "reviewBasis": "Read both changed TypeScript files, the seeded bug report, behavioral gate log and tool trace. Read-only probes of the captured functions are saved in advisory-endpoint-probes.json; they do not rerun the model or change the fixed verdict." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:28.428Z", "reviewId": "025", "cellId": "r1-saluki-cuda-developer-fix-squisq-bugs", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "scenarioId": "fix-squisq-bugs", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "baselineCellId": "r1-saluki-cuda-developer-fix-squisq-bugs", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b9d0d4920e09067c823368552905b92ef33595b5b44e4c9b4ebf41094c86c1b9", "evidenceSha256": "8b3a6a80a5650821002d105c6d3e13cfca11532db962f333b59d96bf2de94ddd", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "bytes": 5556, "sha256": "2a0696ebc7b8c08fa9f0faf52cde78c3d7a54f7caf37ab6aa7f5f60dba8eb41e", "shownBytes": 5556, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "Geohash.ts shows slerp-based getGeohashPath sized by haversine distance, and both neighbor/path functions return []/'' on bad length. However, the longitude slerp is dubious: it mixes degrees with radians (360 vs Math.PI) and interpolates lng linearly with sin weights, so it is likely incorrect.", "limitations": "Only Geohash.ts was shown. persistence.ts (assertImageEditDoc) and Haversine.ts were not provided, so the second bug and the untouched-Haversine constraint can't be checked. No tsc or runtime results were observed.", "meanScore": null, "durationMs": 14608, "costUsd": 0.02071, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4369, "cache_read_input_tokens": 700, "output_tokens": 316, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4369, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 316, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 4369, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4369 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 316, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 4369, "webSearchRequests": 0, "costUSD": 0.02071, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4369, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - fix-squisq-bugs / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 14m 23s; budget used: 27.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 167385`; `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":7790,\"score\":5,\"scoreMax\":5,\"signals\":[\"great-circle-path\",\"graceful-errors\",\"deep-layer-validation\",\"haversine-untouched\",\"tsc-clean\"],\"fa...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.2764246185150115`; `facts.toolUse.totalToolCalls: 14`; `facts.timing.timeToFirstArtifactMs: 167385`; `facts.outcome.durationMs: 862928` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12957 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31534 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.66 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 45183 + 6054 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 22% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 167385`\n- `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":7790,\"score\":5,\"scoreMax\":5,\"signals\":[\"great-circle-path\",\"graceful-errors\",\"deep-layer-validation\",\"haversine-untouched\",\"tsc-clean\"],\"fa...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.2764246185150115`\n- `facts.toolUse.totalToolCalls: 14`\n- `facts.timing.timeToFirstArtifactMs: 167385`\n- `facts.outcome.durationMs: 862928`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 167385`; `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":7790,\"score\":5,\"scoreMax\":5,\"signals\":[\"great-circle-path\",\"graceful-errors\",\"deep-layer-validation\",\"haversine-untouched\",\"tsc-clean\"],\"fa...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.2764246185150115`; `facts.toolUse.totalToolCalls: 14`; `facts.timing.timeToFirstArtifactMs: 167385`; `facts.outcome.durationMs: 862928`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nRead both changed TypeScript files, the seeded bug report, behavioral gate log and tool trace. Read-only probes of the captured functions are saved in advisory-endpoint-probes.json; they do not rerun the model or change the fixed verdict.\n\nStatus: **notApplicable**. No numerical qualitative axes are invented.\n\nNo declared qualitative axes. The five fixed checks pass, but independent endpoint probes show that the saved spherical interpolation is still incorrect. Graceful empty returns and per-layer field validation are present. A native transport failure recovered automatically before completion.\n\nFixed composite remains **9.5/10**; the advice below does not alter the fixed scores or campaign identity.\n\n## Positives\n\n- All five standard signals passed, including the behavioral antimeridian, graceful-error and deep-layer checks, untouched Haversine helper and clean TypeScript compilation (sniff.latest.signals; outcome.reason; log.txt at 16:05:23–24Z).\n- Both invalid-length geohash utilities now return empty values, and each layer is checked for object, id, supported type and numeric coordinates (artifacts.otherFileCount; captured Geohash.ts and persistence.ts). The standard queued continuation recovered the native interruption and reached success without a manual retry (outcome.success; engineContext.launches: 2 in summary).\n\n## Negatives\n\n- The final interpolation is still mathematically wrong despite the five-signal pass (sniff.latest.score: 5/5; advisory-endpoint-probes.json). For decoded endpoints near (40,-74) and (51,0), the returned start is approximately (44.528,0.0055) and end (36.828,-73.9984). Source inspection shows reversed interpolation weights, a missing inverse sine/Cartesian conversion, and radians mixed with the degree constant 360. Even the equatorial antimeridian path reverses endpoints.\n- One run_installed_script call tried the package.json script typecheck as a Gezel project-installed script and failed (toolUse.byTool.run_installed_script: 1; advisory-tool-evidence.json). This is a tool-contract mismatch, not missing TypeScript dependencies; the harness compiled successfully.\n- A /v1/chat/completions fetch failure at 16:02:56.005Z preceded the supervisor’s memory-pressure release at 16:02:56.482Z. Native exit 3221226505 was logged expected=true/memory-pressure at 16:02:57.294Z, followed by relaunch and readiness (perf.systemMemory.peakUsedMb: 31534; advisory-native-recovery.json). Causality remains unresolved; no explicit CUDA OOM was found.\n\n## Tactical experiment — not applied\n\nIn a future version of ../gilde/data/gezel-templates/de/developer/versions/1.5.1/about.md, add: “For numerical repairs, check units and defining invariants before declaring completion; interpolation must return each endpoint at its corresponding parameter boundary.” Verify on geometry and an unrelated financial interpolation task. Separately clarify in the role’s tool guidance that package.json scripts require a process execution surface, not run_installed_script.\n\n## Strategic experiment — not applied\n\nIn packages/inference/src/providers/native/supervisor.ts, add incident correlation for request failure time, pending memory request, release initiation and process termination before expectedExitReason is applied. The same ordering occurred in docblocks-theme-roundtrip. A future controlled stress test should establish whether release ordering or an earlier native crash causes the endpoint failures; do not label this established OOM. For code quality, prototype a generic property-check step in developer workflows that validates endpoint, reversibility and unit invariants beyond a supplied example.\n\n## Performance and fit\n\nDuration 14.38 minutes; recorded decode 8.66 tokens/sec, prefill 396.17 tokens/sec, 6054 generated tokens. Peak VRAM 10630/12227 MiB, process RSS 12957 MiB, whole-system RAM 31534/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57 CUDA; 2 launch(es), 65536 context, one slot, f16 KV, 4096 reasoning budget. Planner: 39/64 dense FFN layers on GPU and 25 on CPU. This is mixed CPU/GPU residency, not full GPU residency. Shared GPU memory and paging are not separately measured. The recovered incident is recorded above; no explicit CUDA OOM or context-overflow message was found.\n\n## Generalization and next experiment\n\nThe earlier symptom-debug task made a compact correct boundary repair; this numerical repair passes its recorded checks yet fails basic endpoint invariants. Native recovery repeats the DocBlocks roundtrip incident across an unrelated coding workload. Both findings warrant generic verification/runtime experiments, not scenario-specific answers or model substitution.\n\nAll suggestions are prospective; no prompts, gates, tuning, task artifacts or model files were edited. Retain this frozen campaign as the baseline and evaluate any later change on at least one unrelated domain before calling it a general improvement.\n\n## Last three same-scenario trials\n\n- [fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/facts.json): pass, fixed 8.5, 455.5 seconds, recorded decode 132.02 tokens/sec. all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)\n- [fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht/facts.json): pass, fixed 9.5, 68.6 seconds, recorded decode 139.43 tokens/sec. all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)\n- [fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-31-32-871Z-smwx/facts.json): pass, fixed 9.5, 159.2 seconds, recorded decode 137.13 tokens/sec. all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)\n\nThese are historical comparisons, not a controlled native-version or model experiment: source/model/native identities may differ. Earlier outcomes: 3/3 passed.\n" }, { "cellId": "r1-saluki-cuda-developer-dev-craftbook-routing", "baselineCellId": "r1-saluki-cuda-developer-dev-craftbook-routing", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T16:08:27.564Z", "finishedAt": "2026-11-04T16:09:44.373Z", "durationMs": 76809, "success": true, "reason": "task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.024604484410657105 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 1 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 76809 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" }, "facts": { "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 12, "peakRssMb": 9765, "peakCpuPercent": 1210.6, "firstAtMs": 1, "lastAtMs": 65158 }, "gpu": { "available": true, "sampleCount": 12, "peakUtilPercent": 98, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 12, "peakUsedMb": 27724, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11817, "totalOutputTokens": 210, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T16:09:40.327Z" } }, "source": "engine-log" }, "derived": { "durationMs": 66383, "meanTokensPerSec": 9.23, "genTokensPerSec": 9.23, "genTokens": 210, "promptTokensPerSec": 644.34, "peakRssMb": 9765, "peakGpuUtilPercent": 98, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "batch-exporter/1", "stepId": "reproduce", "activatedAt": "2026-11-04T16:09:35.458Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T16:08:33.387Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T16:08:33.288Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "dagny": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1535491943359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1535491943359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "durationMs": 76809, "timeoutMs": 3121748, "budgetUsedFraction": 0.024604484410657105 }, "timing": { "startedAt": "2026-11-04T16:08:27.564Z", "finishedAt": "2026-11-04T16:09:44.373Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 44061, "firstTurnTtftMs": 25716, "timeToFirstToolCallMs": 68076 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 1, "byTool": { "start_work": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 3 }, "sniff": { "progression": [ { "atMs": 7814, "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } ], "latest": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T16:08:27.567Z] [trial] id=dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe scenario=dev-craftbook-routing model=underdog-saluki-27b-iq2-mix", "[2026-11-04T16:08:27.568Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "[2026-11-04T16:08:27.568Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-dev-craftbook-routing-Dr64n2", "[2026-11-04T16:08:27.568Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T16:08:27.568Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T16:08:27.571Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T16:08:27.575Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T16:08:27.581Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-dev-craftbook-routing-Dr64n2", "[2026-11-04T16:08:27.581Z] [trial] throughput-scaled ceiling: 25m → 52m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T16:08:33.375Z] [trial] daemon spawned pid=14664 port=https://127.0.0.1:60081", "[2026-11-04T16:08:34.770Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T16:08:34.786Z] [trial] running scenario setup hook", "[2026-11-04T16:08:34.786Z] [trial] meester=dagny", "[2026-11-04T16:08:35.074Z] [trial] scenario setup complete", "[2026-11-04T16:08:35.076Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T16:08:35.150Z] [poll] initial digests hard=75c8da404ab4ef6e soft=f3e4140918f4b373", "[2026-11-04T16:08:35.448Z] [poll] hard-progress: 75c8da404ab4ef6e → 1dc3e352016a58cf after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/3B)", "[2026-11-04T16:09:40.119Z] [poll] terminal: success=true reason=task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "[2026-11-04T16:09:40.356Z] [capture] dumped 1 session(s)", "[2026-11-04T16:09:40.360Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T16:09:40.430Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T16:09:44.372Z] [trial] daemon shut down", "[2026-11-04T16:09:44.380Z] [trial] PASS duration=76809ms reason=task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored" ] }, "metrics": { "process": { "sampleCount": 12, "peakRssMb": 9765, "peakCpuPercent": 1210.6, "firstAtMs": 1, "lastAtMs": 65158 }, "gpu": { "available": true, "sampleCount": 12, "peakUtilPercent": 98, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 12, "peakUsedMb": 27724, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11817, "totalOutputTokens": 210, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T16:09:40.327Z" } }, "source": "engine-log" }, "derived": { "durationMs": 66383, "meanTokensPerSec": 9.23, "genTokensPerSec": 9.23, "genTokens": 210, "promptTokensPerSec": 644.34, "peakRssMb": 9765, "peakGpuUtilPercent": 98, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T16:08:27.564Z", "finishedAt": "2026-11-04T16:09:44.373Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 44061, "firstTurnTtftMs": 25716, "timeToFirstToolCallMs": 68076 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "batch-exporter/1", "stepId": "reproduce", "activatedAt": "2026-11-04T16:09:35.458Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T16:08:33.387Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T16:08:33.288Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "dagny": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1535491943359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1535491943359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-dev-craftbook-routing", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "scenarioId": "dev-craftbook-routing", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "artifactPath": null, "artifactBytes": 0, "artifactSha256": null, "artifacts": [], "axisSource": null, "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "This selection-only probe successfully started the bundled bug-fix-tdd workflow. No authored deliverable was required or produced; seeded code is not a generated artifact. No qualitative score is assigned, and selection success does not establish bug-fix completion.", "reviewBasis": "Read the start_work event, scenario contract, final log and frozen facts/score. The scenario explicitly terminates at matching task creation. The captured seeded files are not judged as generated work." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:37.387Z", "reviewId": "026", "cellId": "r1-saluki-cuda-developer-dev-craftbook-routing", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "scenarioId": "dev-craftbook-routing", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "baselineCellId": "r1-saluki-cuda-developer-dev-craftbook-routing", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "5a112a652864a89923d9de1c0dfd533846e98177d89a772a31c21fc3bfed5744", "evidenceSha256": "64c9c8db497879f22f0fee5a0b06f8cc120a0c823fa10ab8ec7634778f562411", "artifactFiles": [], "contextFiles": [], "scoreAxes": {}, "justification": "No deliverable was present at closeout: artifacts is empty, declaredAxes is empty, and no missing paths are listed. The run is marked runner-terminal with noArtifact. The evidence has no transcript, tool calls or workspace state showing whether a recipe was picked or a task started, so the missing output can't be explained.", "limitations": "Only metadata was supplied, with no logs, context files or task state. I can't tell whether the agent chose a recipe, started the task or stopped early. No score is given, and this is neither a model failure nor a success. Nothing was run or retried.", "meanScore": null, "durationMs": 14291, "costUsd": 0.009216, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1523, "cache_read_input_tokens": 700, "output_tokens": 305, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1523, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 305, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1523, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1523 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 305, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1523, "webSearchRequests": 0, "costUSD": 0.009216, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1523, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - dev-craftbook-routing / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.8 / 10** (band: ship-ready)\n**Outcome: success** (duration: 1m 17s; budget used: 2.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 5 / 10 | The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.024604484410657105`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 76809` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **8.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9765 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 98% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27724 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.23 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 11817 + 210 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 50% of its measured signals.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.024604484410657105`\n- `facts.toolUse.totalToolCalls: 1`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 76809`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.024604484410657105`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 76809`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nRead the start_work event, scenario contract, final log and frozen facts/score. The scenario explicitly terminates at matching task creation. The captured seeded files are not judged as generated work.\n\nStatus: **noArtifact**. No numerical qualitative axes are invented.\n\nThis selection-only probe successfully started the bundled bug-fix-tdd workflow. No authored deliverable was required or produced; seeded code is not a generated artifact. No qualitative score is assigned, and selection success does not establish bug-fix completion.\n\nFixed composite remains **8.8/10**; the advice below does not alter the fixed scores or campaign identity.\n\n## Positives\n\n- One start_work call selected craftbook:bug-fix-tdd and created batch-exporter/1 without authoring a new recipe (toolUse.byTool.start_work: 1; outcome.reason; recording/chat-events.jsonl at 16:09:35.644Z). This matches a known non-urgent defect with a requested failing test.\n- The selection completed in 76.8 seconds with 210 generated tokens and no fixed red flag or missing role (outcome.durationMs; perf.derived.genTokens; toolUse.redFlags; team.missingExpectedRoles).\n\n## Negatives\n\n- The latest sniff remains at 1/2 with an obsolete “no task has been started” message although the terminal success check sees the created task (sniff.latest versus outcome.success). Fixed quality is 5 and composite 8.8; these frozen values are preserved. This is a snapshot inconsistency, not evidence that the correct recipe was not selected.\n- No failing test or repair is established by this trial (timing.timeToFirstArtifactMs: null; toolUse.byTool contains start_work only). The standard runner logged an automatic paused-task retry after terminal success at 16:09:40.192Z, while facts.autoAnswer.total is 0; execution was captured and shut down immediately. No monitoring action retried it.\n\n## Tactical experiment — not applied\n\nIn packages/mcp/src/server.ts, where the start_work response calls taskPersonNoun(created), use an output-neutral fallback such as “the work is underway” for engineering recipes instead of “your document is underway”. The captured response labels a bug-fix task a document; validate this wording change on document and coding routing without altering selection instructions.\n\n## Strategic experiment — not applied\n\nIn evals/src/craftbooks/authoring/route-dev.ts and the shared terminal capture path, retain a timestamped terminal success-check snapshot beside the last polling sniff, and stop post-terminal auto-answer work before teardown. Preserve current fixed scores; verify the logging improvement on unrelated selection probes. This resolves observable state ordering without weakening any capability condition.\n\n## Performance and fit\n\nDuration 1.28 minutes; recorded decode 9.23 tokens/sec, prefill 644.34 tokens/sec, 210 generated tokens. Peak VRAM 10628/12227 MiB, process RSS 9765 MiB, whole-system RAM 27724/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57 CUDA; 1 launch(es), 65536 context, one slot, f16 KV, 4096 reasoning budget. Planner: 39/64 dense FFN layers on GPU and 25 on CPU. This is mixed CPU/GPU residency, not full GPU residency. Shared GPU memory and paging are not separately measured. No explicit OOM, context overflow or unexpected native/transport incident was found in this trial.\n\n## Generalization and next experiment\n\nInterface-contract executes and compiles produced code, whereas this trial measures recipe choice only. Both are useful developer signals, but only the former demonstrates completed output. The contrast prevents a routing success from being counted as a full engineering workflow success.\n\nAll suggestions are prospective; no prompts, gates, tuning, task artifacts or model files were edited. Retain this frozen campaign as the baseline and evaluate any later change on at least one unrelated domain before calling it a general improvement.\n\n## Last three same-scenario trials\n\n- [dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/facts.json): pass, fixed 8.8, 72.7 seconds, recorded decode 110.02 tokens/sec. task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored\n- [dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n/facts.json): fail, fixed 2, 324.4 seconds, recorded decode 112 tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook selection (engineering); missing signals stayed unchanged: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it, a new craftbook was authored (regression-test-and-fix) — the library already covers this; use the existing recipe instead. Last failure: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it\n- [dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-43-02-884Z-pjd9/facts.json): fail, fixed 2, 274.2 seconds, recorded decode 111.46 tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook selection (engineering); missing signals stayed unchanged: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it, a new craftbook was authored (regression-test-and-fix) — the library already covers this; use the existing recipe instead. Last failure: no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it\n\nThese are historical comparisons, not a controlled native-version or model experiment: source/model/native identities may differ. Earlier outcomes: 1/3 passed.\n" }, { "cellId": "r1-saluki-cuda-developer-interface-contract", "baselineCellId": "r1-saluki-cuda-developer-interface-contract", "suite": "developer", "scenario": "interface-contract", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T16:12:00.767Z", "finishedAt": "2026-11-04T16:15:21.688Z", "durationMs": 200921, "success": true, "reason": "tsc emit clean + node integration.mjs passed: 6 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 142441 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "interface-contract", "bytes": 1365, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.053634742070282196 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 142441 }, { "source": "facts", "path": "outcome.durationMs", "value": 200921 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 33, "peakRssMb": 10852, "peakCpuPercent": 1205.8, "firstAtMs": 0, "lastAtMs": 189292 }, "gpu": { "available": true, "sampleCount": 33, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 27791, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 14715, "totalOutputTokens": 1334, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7612, "todayTokensCached": 7067, "todayTokensOut": 129, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7612, "totalTokensCached": 7067, "totalTokensOut": 129, "totalCost": 0, "medianOutputTokensPerSec": 9.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.4, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7612, "cachedInputTokens": 7067, "outputTokens": 129, "cost": 0, "durationMs": 91969, "outputTokensPerSec": 9.441312543071378, "at": "2026-11-04T16:13:42.150Z" }, "lastUpdated": "2026-11-04T16:13:42.150Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T16:15:17.808Z" } }, "source": "engine-log" }, "derived": { "durationMs": 191254, "meanTokensPerSec": 9.41, "genTokensPerSec": 9.41, "genTokens": 1334, "promptTokensPerSec": 577, "peakRssMb": 10852, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7612, "output": 129 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "event-pipeline/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-04T16:13:26.055Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "bab1607e-55f2-4e88-b391-578b05eb9505" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T16:12:06.473Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T16:12:06.395Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "aldric": 1, "lyubov": 1 }, "perTask": { "event-pipeline/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 3, "maxContextFill": 0.101318359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.101318359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tsc emit clean + node integration.mjs passed: 6 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "durationMs": 200921, "timeoutMs": 3746098, "budgetUsedFraction": 0.053634742070282196 }, "timing": { "startedAt": "2026-11-04T16:12:00.767Z", "finishedAt": "2026-11-04T16:15:21.688Z", "timeToFirstArtifactMs": 142441, "timeToLastArtifactWriteMs": 195574, "timeToFirstTokenMs": 28140, "firstTurnTtftMs": 18726, "timeToFirstToolCallMs": 85343 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 4, "byTool": { "start_work": 1, "write_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 196702, "filePath": "interface-contract", "bytes": 1365, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } ], "latest": { "filePath": "interface-contract", "bytes": 1365, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T16:12:00.770Z] [trial] id=interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i scenario=interface-contract model=underdog-saluki-27b-iq2-mix", "[2026-11-04T16:12:00.770Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "[2026-11-04T16:12:00.770Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-interface-contract-VBZnq1", "[2026-11-04T16:12:00.771Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T16:12:00.771Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T16:12:00.773Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T16:12:00.777Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T16:12:00.782Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-interface-contract-VBZnq1", "[2026-11-04T16:12:00.782Z] [trial] throughput-scaled ceiling: 30m → 62m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T16:12:06.502Z] [trial] daemon spawned pid=45508 port=https://127.0.0.1:58182", "[2026-11-04T16:12:07.386Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T16:12:07.402Z] [trial] meester=aldric", "[2026-11-04T16:12:07.548Z] [trial] sent prompt to meester (1435 chars)", "[2026-11-04T16:12:07.605Z] [poll] initial digests hard=ff8e34d342ae7858 soft=f742d8a40968ff44", "[2026-11-04T16:13:30.187Z] [poll] hard-progress: ff8e34d342ae7858 → 63f06b4b81083c9e after 83s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=181,stream=862) sniff=0/0B)", "[2026-11-04T16:13:47.827Z] [poll] hard-progress: 63f06b4b81083c9e → 85d4de09584b35e5 after 18s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=1,slot=219,stream=971) sniff=0/0B)", "[2026-11-04T16:14:23.527Z] [poll] hard-progress: 85d4de09584b35e5 → 86ac721746256b70 after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=282,stream=1199) sniff=1/0B)", "[2026-11-04T16:15:17.474Z] [poll] terminal: success=true reason=tsc emit clean + node integration.mjs passed: 6 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "[2026-11-04T16:15:17.835Z] [capture] dumped 2 session(s)", "[2026-11-04T16:15:17.839Z] [capture] snapshotted telemetry for 2 session(s)", "[2026-11-04T16:15:17.908Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T16:15:21.686Z] [trial] daemon shut down", "[2026-11-04T16:15:21.696Z] [trial] PASS duration=200921ms reason=tsc emit clean + node integration.mjs passed: 6 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)" ] }, "metrics": { "process": { "sampleCount": 33, "peakRssMb": 10852, "peakCpuPercent": 1205.8, "firstAtMs": 0, "lastAtMs": 189292 }, "gpu": { "available": true, "sampleCount": 33, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 27791, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 14715, "totalOutputTokens": 1334, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7612, "todayTokensCached": 7067, "todayTokensOut": 129, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7612, "totalTokensCached": 7067, "totalTokensOut": 129, "totalCost": 0, "medianOutputTokensPerSec": 9.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.4, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7612, "cachedInputTokens": 7067, "outputTokens": 129, "cost": 0, "durationMs": 91969, "outputTokensPerSec": 9.441312543071378, "at": "2026-11-04T16:13:42.150Z" }, "lastUpdated": "2026-11-04T16:13:42.150Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T16:15:17.808Z" } }, "source": "engine-log" }, "derived": { "durationMs": 191254, "meanTokensPerSec": 9.41, "genTokensPerSec": 9.41, "genTokens": 1334, "promptTokensPerSec": 577, "peakRssMb": 10852, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7612, "output": 129 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T16:12:00.767Z", "finishedAt": "2026-11-04T16:15:21.688Z", "timeToFirstArtifactMs": 142441, "timeToLastArtifactWriteMs": 195574, "timeToFirstTokenMs": 28140, "firstTurnTtftMs": 18726, "timeToFirstToolCallMs": 85343 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "event-pipeline/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-04T16:13:26.055Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "bab1607e-55f2-4e88-b391-578b05eb9505" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T16:12:06.473Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T16:12:06.395Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "aldric": 1, "lyubov": 1 }, "perTask": { "event-pipeline/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 3, "maxContextFill": 0.101318359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.101318359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-interface-contract", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "scenarioId": "interface-contract", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/workspace/event-pipeline/src/types.ts", "artifactBytes": 1367, "artifactSha256": "0a51c5abc6308e5cd4ac3063688969109ef73eb296bd4c4d1e5099751da0503f", "artifacts": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/workspace/event-pipeline/src/types.ts", "bytes": 297, "sha256": "0a51c5abc6308e5cd4ac3063688969109ef73eb296bd4c4d1e5099751da0503f" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/workspace/event-pipeline/src/producer.ts", "bytes": 674, "sha256": "b1d0ec110500f7199b2965567421eb5fe88a80f60b57b38228e3f1b9c20a0274" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/workspace/event-pipeline/src/consumer.ts", "bytes": 396, "sha256": "42871f3117dcf4784892f55a27065e0374ca428e6f0d8c70291595e3c1f461ee" } ], "axisSource": null, "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared qualitative axes. Three concise files compile and route all six supplied events across four kinds. Independent probes show counts fail for constructor and __proto__ because the accumulator is a normal object. One worker completed the code; coupled multi-worker coordination was not exercised.", "reviewBasis": "Read all three captured TypeScript files and the original contract. Checked arbitrary string-kind behavior with TypeScript transpilation matching the standard gate; direct Node type stripping does not elide ordinary type imports, so it was not used as the verdict. Evidence is in advisory-key-probes.json." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:42.864Z", "reviewId": "027", "cellId": "r1-saluki-cuda-developer-interface-contract", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "scenarioId": "interface-contract", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "baselineCellId": "r1-saluki-cuda-developer-interface-contract", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "2b4377de31a2e0f684c474b075686c0e42dd1e0107936a9c438b5625507a0134", "evidenceSha256": "6ce4b54b83bba30dc1b2ecabd42d8594c71b867470d21943d92d4dd5fa60c8bb", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/workspace/event-pipeline/src/types.ts", "bytes": 297, "sha256": "0a51c5abc6308e5cd4ac3063688969109ef73eb296bd4c4d1e5099751da0503f", "shownBytes": 297, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes are declared. Only src/types.ts is shown. It looks correct: EventRecord has kind and payload, and Sink has handle(kind, payload): void, so it matches the contract. producer.ts and consumer.ts are absent from the evidence, so completeness and the tsc and integration checks can't be judged.", "limitations": "Source-only review of one of the three required files. There is no evidence of producer.ts, consumer.ts, the project setup, the .ts import extensions, tsc results or integration results. The executionDisposition is runner-terminal, but no outcome is shown.", "meanScore": null, "durationMs": 14405, "costUsd": 0.010854, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1920, "cache_read_input_tokens": 700, "output_tokens": 310, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1920, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 310, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1920, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1920 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 310, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1920, "webSearchRequests": 0, "costUSD": 0.010854, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1920, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - interface-contract / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 3m 21s; budget used: 5.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 142441`; `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1365,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.053634742070282196`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: 142441`; `facts.outcome.durationMs: 200921` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10852 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27791 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.41 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 14715 + 1334 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 10% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 142441`\n- `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1365,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.053634742070282196`\n- `facts.toolUse.totalToolCalls: 4`\n- `facts.timing.timeToFirstArtifactMs: 142441`\n- `facts.outcome.durationMs: 200921`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 142441`; `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1365,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.053634742070282196`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: 142441`; `facts.outcome.durationMs: 200921`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nRead all three captured TypeScript files and the original contract. Checked arbitrary string-kind behavior with TypeScript transpilation matching the standard gate; direct Node type stripping does not elide ordinary type imports, so it was not used as the verdict. Evidence is in advisory-key-probes.json.\n\nStatus: **notApplicable**. No numerical qualitative axes are invented.\n\nNo declared qualitative axes. Three concise files compile and route all six supplied events across four kinds. Independent probes show counts fail for constructor and __proto__ because the accumulator is a normal object. One worker completed the code; coupled multi-worker coordination was not exercised.\n\nFixed composite remains **10/10**; the advice below does not alter the fixed scores or campaign identity.\n\n## Positives\n\n- The shared EventRecord/Sink contract is imported by producer and consumer, and the harness compiled and routed all six events across four kinds (sniff.latest.signals: files-present,tsc-clean,integration-pass; outcome.reason). Three writes produced the required files without tool errors (toolUse.byTool.write_file: 3).\n- Completion took 200.9 seconds and 1334 generated tokens with no fixed red flag or missing role (outcome.durationMs; perf.derived.genTokens; toolUse.redFlags; team.missingExpectedRoles). The source is short and the public payload type is safely unknown rather than any.\n\n## Negatives\n\n- Counts use a normal {} accumulator even though kind is any string. Read-only probes show constructor becomes a function-string-plus-1 and __proto__ is not counted as an own property, although sink.handle receives both events (sniff.latest.score: 5/5 records the narrower pass; advisory-key-probes.json records the extra checks).\n- The run used one non-Meester session, so it demonstrates a coherent solo implementation rather than independent producer/consumer worker coordination (outcome.reason: team shape solo; team.rolesCreated). This is allowed by the prompt’s “consider splitting” wording and is not a deterministic failure.\n\n## Tactical experiment — not applied\n\nIn a future version of ../gilde/data/gezel-templates/bu/builder/versions/1.5.1/about.md, add: “When external strings become dictionary keys, use Map or a null-prototype object and verify reserved object-property names.” Test on event aggregation and unrelated CSV group-by tasks; do not insert scenario-specific filenames or expected event values.\n\n## Strategic experiment — not applied\n\nPrototype an opt-in adversarial-input verification step in generic developer craftbooks that derives edge classes from public types (arbitrary string keys, empty collections, endpoint values) and records actual execution results. Implement its deterministic helper under packages/script-stdlib/src and route through the existing script boundary. Measure defect discovery across counting and numerical tasks without changing the running eval gates.\n\n## Performance and fit\n\nDuration 3.35 minutes; recorded decode 9.41 tokens/sec, prefill 577 tokens/sec, 1334 generated tokens. Peak VRAM 10626/12227 MiB, process RSS 10852 MiB, whole-system RAM 27791/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Native 0.1.57 CUDA; 1 launch(es), 65536 context, one slot, f16 KV, 4096 reasoning budget. Planner: 39/64 dense FFN layers on GPU and 25 on CPU. This is mixed CPU/GPU residency, not full GPU residency. Shared GPU memory and paging are not separately measured. No explicit OOM, context overflow or unexpected native/transport incident was found in this trial.\n\n## Generalization and next experiment\n\nThis counter and the geohash repair both pass fixed execution checks while missing broad public-contract invariants. A reusable verification step should cover them without hard-coding their outputs. The earlier data-wrangle failure additionally shows why a usable script execution surface is necessary for model-led verification.\n\nAll suggestions are prospective; no prompts, gates, tuning, task artifacts or model files were edited. Retain this frozen campaign as the baseline and evaluate any later change on at least one unrelated domain before calling it a general improvement.\n\n## Last three same-scenario trials\n\n- [interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/facts.json): pass, fixed 9.5, 52.7 seconds, recorded decode 126.37 tokens/sec. tsc emit clean + node integration.mjs passed: 5 event(s) across 3 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)\n- [interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw/facts.json): pass, fixed 10, 49.8 seconds, recorded decode 115.11 tokens/sec. tsc emit clean + node integration.mjs passed: 5 event(s) across 3 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)\n- [interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-54-19-694Z-2ift/facts.json): pass, fixed 10, 52.6 seconds, recorded decode 124.85 tokens/sec. tsc emit clean + node integration.mjs passed: 5 event(s) across 3 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)\n\nThese are historical comparisons, not a controlled native-version or model experiment: source/model/native identities may differ. Earlier outcomes: 3/3 passed.\n" }, { "cellId": "r1-saluki-cuda-developer-craftbook-code-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-code-review", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T16:17:39.750Z", "finishedAt": "2026-11-04T16:44:13.436Z", "durationMs": 1593686, "success": true, "reason": "craftbook-code-review passed 4 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1416594 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-code-review", "bytes": 3500, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.4254256028539563 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 14 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1416594 }, { "source": "facts", "path": "outcome.durationMs", "value": 1593686 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 266, "peakRssMb": 14135, "peakCpuPercent": 1208.4, "firstAtMs": 0, "lastAtMs": 1579995 }, "gpu": { "available": true, "sampleCount": 266, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 266, "peakUsedMb": 30434, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 52197, "totalOutputTokens": 11499, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 38395, "todayTokensCached": 34950, "todayTokensOut": 4334, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 38395, "totalTokensCached": 34950, "totalTokensOut": 4334, "totalCost": 0, "medianOutputTokensPerSec": 8.1, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.1, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13295, "cachedInputTokens": 12990, "outputTokens": 175, "cost": 0, "durationMs": 115275, "outputTokensPerSec": 8.373086647443536, "at": "2026-11-04T16:43:15.011Z" }, "lastUpdated": "2026-11-04T16:43:15.011Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T16:44:06.828Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1581473, "meanTokensPerSec": 8.18, "genTokensPerSec": 8.18, "genTokens": 11499, "promptTokensPerSec": 426.6, "peakRssMb": 14135, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 38395, "output": 4334 } } } }, "craftbook": { "craftbookId": "code-review", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 2, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "code-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T16:17:46.630Z", "completedAt": "2026-11-04T16:21:03.569Z", "ms": 196939, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "3d44ecf9-dd4f-4379-890e-1a9294cb85f7" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-04T16:21:03.571Z", "completedAt": "2026-11-04T16:41:16.803Z", "ms": 1213232, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "3d44ecf9-dd4f-4379-890e-1a9294cb85f7" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "done", "activatedAt": "2026-11-04T16:41:16.805Z", "completedAt": "2026-11-04T16:43:14.970Z", "ms": 118165, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "3d44ecf9-dd4f-4379-890e-1a9294cb85f7" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T16:17:45.395Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T16:17:45.299Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 196939 }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "simona": 2 }, "perTask": { "code-review-workflow-eval/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 2, "continuityBreaks": 0, "sessionsPerStep": 0.2, "maxMessages": 6, "maxContextFill": 0.177276611328125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.177276611328125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-code-review passed 4 deterministic craftbook checks", "durationMs": 1593686, "timeoutMs": 3746098, "budgetUsedFraction": 0.4254256028539563 }, "timing": { "startedAt": "2026-11-04T16:17:39.750Z", "finishedAt": "2026-11-04T16:44:13.436Z", "timeToFirstArtifactMs": 1416594, "timeToLastArtifactWriteMs": 1416594, "timeToFirstTokenMs": 31727, "firstTurnTtftMs": 18546, "timeToFirstToolCallMs": 40661 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 14, "byTool": { "read_artifact": 3, "read_file": 5, "write_task_note": 2, "advance_task_step": 2, "write_artifact": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 15 }, "sniff": { "progression": [ { "atMs": 7092, "filePath": "craftbook-code-review", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, src/payment.js, docs/review-conventions.md" }, { "atMs": 94923, "filePath": "craftbook-code-review", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: src/payment.js" }, { "atMs": 1420007, "filePath": "craftbook-code-review", "bytes": 3500, "score": 2, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: src/payment.js" }, { "atMs": 1538862, "filePath": "craftbook-code-review", "bytes": 3500, "score": 3, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: src/payment.js" }, { "atMs": 1586575, "filePath": "craftbook-code-review", "bytes": 3500, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-code-review", "bytes": 3500, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T16:17:39.754Z] [trial] id=craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0 scenario=craftbook-code-review model=underdog-saluki-27b-iq2-mix", "[2026-11-04T16:17:39.754Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T16:17:39.754Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "[2026-11-04T16:17:39.754Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T16:17:39.754Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-code-review-vZgIck", "[2026-11-04T16:17:39.757Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T16:17:39.761Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T16:17:39.768Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-code-review-vZgIck", "[2026-11-04T16:17:39.768Z] [trial] throughput-scaled ceiling: 30m → 62m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T16:17:45.343Z] [trial] daemon spawned pid=37812 port=https://127.0.0.1:54212", "[2026-11-04T16:17:46.179Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T16:17:46.195Z] [trial] running scenario setup hook", "[2026-11-04T16:17:46.195Z] [trial] meester=vanya", "[2026-11-04T16:17:46.754Z] [trial] scenario setup complete", "[2026-11-04T16:17:46.755Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T16:17:46.807Z] [poll] initial digests hard=4bab42f37c1f4f82 soft=7c78a0bd0d7942b9", "[2026-11-04T16:17:46.893Z] [poll] hard-progress: 4bab42f37c1f4f82 → 874804517bd5670f after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T16:17:52.042Z] [poll] hard-progress: 874804517bd5670f → eb9a401d786515ee after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T16:18:21.605Z] [poll] hard-progress: eb9a401d786515ee → 0e95acf09fcd83b7 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=30,stream=67) sniff=1/0B)", "[2026-11-04T16:18:33.410Z] [poll] hard-progress: 0e95acf09fcd83b7 → 30946e8f5489c240 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=47,stream=124) sniff=1/0B)", "[2026-11-04T16:19:14.730Z] [poll] hard-progress: 30946e8f5489c240 → cb9676f8303a0219 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=146,stream=427) sniff=1/0B)", "[2026-11-04T16:20:55.250Z] [poll] hard-progress: cb9676f8303a0219 → 8d44185ff6c4fc53 after 101s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=405,stream=1668) sniff=1/0B)", "[2026-11-04T16:21:07.018Z] [poll] hard-progress: 8d44185ff6c4fc53 → 3ffb707310441a98 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=417,stream=1708) sniff=1/0B)", "[2026-11-04T16:31:59.327Z] [poll] hard-progress: 3ffb707310441a98 → 0172b3579928e92c after 652s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=7,slot=2096,stream=7451) sniff=1/0B)", "[2026-11-04T16:32:52.701Z] [poll] hard-progress: 0172b3579928e92c → db5b0317d97881d8 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=9,slot=2212,stream=7804) sniff=1/0B)" ] }, "metrics": { "process": { "sampleCount": 266, "peakRssMb": 14135, "peakCpuPercent": 1208.4, "firstAtMs": 0, "lastAtMs": 1579995 }, "gpu": { "available": true, "sampleCount": 266, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 266, "peakUsedMb": 30434, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 52197, "totalOutputTokens": 11499, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 38395, "todayTokensCached": 34950, "todayTokensOut": 4334, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 38395, "totalTokensCached": 34950, "totalTokensOut": 4334, "totalCost": 0, "medianOutputTokensPerSec": 8.1, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.1, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13295, "cachedInputTokens": 12990, "outputTokens": 175, "cost": 0, "durationMs": 115275, "outputTokensPerSec": 8.373086647443536, "at": "2026-11-04T16:43:15.011Z" }, "lastUpdated": "2026-11-04T16:43:15.011Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T16:44:06.828Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1581473, "meanTokensPerSec": 8.18, "genTokensPerSec": 8.18, "genTokens": 11499, "promptTokensPerSec": 426.6, "peakRssMb": 14135, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 38395, "output": 4334 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T16:17:39.750Z", "finishedAt": "2026-11-04T16:44:13.436Z", "timeToFirstArtifactMs": 1416594, "timeToLastArtifactWriteMs": 1416594, "timeToFirstTokenMs": 31727, "firstTurnTtftMs": 18546, "timeToFirstToolCallMs": 40661 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 2, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "code-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T16:17:46.630Z", "completedAt": "2026-11-04T16:21:03.569Z", "ms": 196939, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "3d44ecf9-dd4f-4379-890e-1a9294cb85f7" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-04T16:21:03.571Z", "completedAt": "2026-11-04T16:41:16.803Z", "ms": 1213232, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "3d44ecf9-dd4f-4379-890e-1a9294cb85f7" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "done", "activatedAt": "2026-11-04T16:41:16.805Z", "completedAt": "2026-11-04T16:43:14.970Z", "ms": 118165, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "3d44ecf9-dd4f-4379-890e-1a9294cb85f7" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T16:17:45.395Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T16:17:45.299Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 196939 }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "simona": 2 }, "perTask": { "code-review-workflow-eval/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 2, "continuityBreaks": 0, "sessionsPerStep": 0.2, "maxMessages": 6, "maxContextFill": 0.177276611328125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.177276611328125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-craftbook-code-review", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "scenarioId": "craftbook-code-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "artifactBytes": 3506, "artifactSha256": "6329ff05a8053844f688a7773c17b3a704e30f0d58f950225280bf83b2023507", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/co/code-review/versions/1.0.4/test.json", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": { "workflow fidelity": 8, "usefulness": 7 }, "meanScore": 7.5, "justification": "The review covers all four main seeded defects, avoids the deliberate house-style traps, and gives concrete fixes with a consistent request-changes verdict. Four payment-file citations are wrong in the saved report, although the refused first draft had them right. Input validation also overlaps the injection findings.", "reviewBasis": "Read the full final report, the refused first draft, complete staged diff, conventions, payment source and task notes. Checked citation line numbers against the captured source. No external security service or live system was used. Model identity visible in paths." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:45.767Z", "reviewId": "028", "cellId": "r1-saluki-cuda-developer-craftbook-code-review", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "scenarioId": "craftbook-code-review", "status": "scored", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "baselineCellId": "r1-saluki-cuda-developer-craftbook-code-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "f91105a88f4f72bbbf23626c99204ba7dc5a4de65363e6bf291118101c21a881", "evidenceSha256": "6c13acd501ef05df1d1273d31108e647646db452cc9b90001b56d570f468380c", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "bytes": 3506, "sha256": "6329ff05a8053844f688a7773c17b3a704e30f0d58f950225280bf83b2023507", "shownBytes": 3506, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0\\workspace\\code-review-workflow-eval\\source\\craftbook-eval-brief.md", "bytes": 1372, "sha256": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "shownBytes": 1372, "truncated": false } ], "scoreAxes": { "workflow fidelity": 7, "usefulness": 8 }, "justification": "The report has the required Summary, a severity-ranked Findings table with file and line citations, and a Verdict. The verdict follows the stated rule that any critical or major finding means request-changes. It names the review kind and head, and it leaves house-style modules unflagged. The scope note and task-note stamp are not shown.", "limitations": "The diff and manifest were not supplied, so I could not check the line citations or the claimed removed code. The scope note, task notes and step advancement are not shown. The review of the deliverable stands on internal consistency and plausibility only.", "meanScore": 7.5, "durationMs": 15637, "costUsd": 0.021309799999999997, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4127, "cache_read_input_tokens": 778, "output_tokens": 472, "output_tokens_details": { "thinking_tokens": 158 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4127, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 472, "cache_read_input_tokens": 778, "cache_creation_input_tokens": 4127, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4127 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 472, "cacheReadInputTokens": 778, "cacheCreationInputTokens": 4127, "webSearchRequests": 0, "costUSD": 0.021309799999999997, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 158, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4127, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-code-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 26m 34s; budget used: 42.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1416594`; `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":3500,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.4254256028539563`; `facts.toolUse.totalToolCalls: 14`; `facts.timing.timeToFirstArtifactMs: 1416594`; `facts.outcome.durationMs: 1593686` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14135 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30434 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.18 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 52197 + 11499 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 197s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.20 | `continuity.sessions` |\n| Sessions reused across steps | 1 (2 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 18% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1416594`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":3500,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.4254256028539563`\n- `facts.toolUse.totalToolCalls: 14`\n- `facts.timing.timeToFirstArtifactMs: 1416594`\n- `facts.outcome.durationMs: 1593686`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1416594`; `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":3500,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.4254256028539563`; `facts.toolUse.totalToolCalls: 14`; `facts.timing.timeToFirstArtifactMs: 1416594`; `facts.outcome.durationMs: 1593686`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nRead the complete 3506-byte report, the first refused draft, all diff hunks, conventions, source line numbers and task notes. The exact declared axes from code-review 1.0.4 test.json are scored separately from the fixed 9.5 composite.\n\n| Axis | Score | Exact description |\n|---|---:|---|\n| workflow fidelity | 8 | The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute. |\n| usefulness | 7 | The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief. |\n\n**Advisory mean: 7.5/10 — codex-inline/current-session.**\n\nThe review covers all four main seeded defects, avoids the deliberate house-style traps, and gives concrete fixes with a consistent request-changes verdict. Four payment-file citations are wrong in the saved report, although the refused first draft had them right. Input validation also overlaps the injection findings.\n\n## Positives\n\n- All four deterministic workflow checks passed, and the attributed craftbook reached its terminal step (outcome.success; sniff.latest.score: 4/4; craftbook.reachedTerminal). The Summary, severity-ranked Findings and request-changes Verdict form a coherent review, followed by a matching verdict note.\n- The report finds the four distinct seeded defect classes: SQL concatenation in payment.js, the pagination end-bound error, settings.js reporting success from its catch, and export.js leaking a handle when a query/write throws (artifacts.otherFileCount; final report and captured changes.diff). It avoids all three documented bait patterns and out-of-scope files, while suggesting concrete parameterization, half-open bounds, explicit failure handling and finally-based cleanup.\n- There were no fixed red flags, missing roles or model-authored source changes (toolUse.redFlags; team.missingExpectedRoles; toolUse.byTool shows report/task-note writes only). The head SHA and seven-file scope are supported by the manifest.\n\n## Negatives\n\n- Four payment citations are incorrect in the final file: SELECT is line 7, not 8; INSERT text is 11, not 13; TODO is 6, not 7; console.log is 9, not 11 (sniff.latest.bytes: 3500; advisory-grounding-refusal.json). The first refused draft had all four correct. The later read of payment.js did not trigger a correction before the trial ended.\n- The first write was refused because the report's internal reference “findings 3–6” was treated as an externally unsupported number (toolUse.byTool.write_artifact: 2; recording/chat-events.jsonl at 16:31:57.252Z). Re-reading and rewriting took another 9m19s before the successful write at 16:41:16.341Z, and the final write still logged a grounding warning. This associates an unnecessary repair loop with citation degradation; it does not prove every part of the elapsed time was caused by the refusal.\n- Fourteen calls and 11499 generated tokens produced a 3506-byte UTF-8 report (the sniff field records 3500); the first saved artifact arrived at 23m37s (toolUse.totalToolCalls; perf.derived.genTokens; timing.timeToFirstArtifactMs: 1416594). The input-validation finding overlaps the injection findings and could distinguish pre-existing validation debt from the newly introduced concatenation more clearly.\n\n## Tactical experiment — not applied\n\nIn a new version of ../gilde/data/gezel-templates/re/reviewer/versions/1.4.1/about.md, add: “Before saving a review, verify each citation against the target file's line numbers, not the diff document's display rows; preserve verified citations during a prose-only repair. Separate new regressions from pre-existing issues.” Measure citation accuracy and unchanged-fact preservation on code reviews and unrelated source-cited document revisions. Do not add fixture filenames or expected findings.\n\n## Strategic experiment — not applied\n\nIn packages/service/src/chat/evidence-ledger.ts, distinguish validated internal document references from external factual numbers: when a sentence cites finding rows, resolve them against the proposed artifact's table before demanding source evidence for their identifiers. Keep ordinary factual-number checks intact and add counterexamples where a purported row does not exist. This same broad problem appeared with derived numbers in the A/B readout and authored structure in the presentation workflow. Measure false refusals, artifact rewrites and source fidelity across at least those unrelated output types.\n\nA second prospective helper under packages/script-stdlib/src could resolve review citations against a staged snapshot and expose mismatches without editing source. This would verify addresses mechanically while leaving semantic review to the model. No runner gate or score was changed here.\n\n## Performance and fit\n\nDuration 26m34s; recorded decode 8.18 tokens/sec and prefill 426.6 tokens/sec, with 11499 generated tokens. Peak VRAM 10630/12227 MiB, process RSS 14135 MiB, whole-system RAM 30434/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). One native 0.1.57 CUDA launch, 65536 context, f16 KV, one slot, 4096 reasoning budget. Planner retained 39/64 dense FFN layers on GPU and 25 on CPU: mixed residency rather than full GPU fit. Shared GPU memory and paging were not separately measured.\n\nNo explicit CUDA OOM, context overflow, endpoint failure or unexpected native exit was found. The logged stopped-turn error follows terminal success and stdin EOF during normal teardown; the native process exits with expectedReason=stop.\n\n## Generalization and next experiment\n\nThis review identifies real defects across four modules, while fix-squisq-bugs and interface-contract demonstrate that passing code generation checks can leave other defects. Together they support a distinction between finding common problems and reliably implementing every public contract. A citation-preserving repair and internal-reference experiment should retain this review's breadth without adding unnecessary rewriting. The repeated grounding-refusal symptom crosses technical reports, A/B analysis and presentation writing, so the proposed evidence-ledger work is not tied to a single eval answer.\n\n## Last three same-scenario trials\n\n- [craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/facts.json): pass, fixed 9.5, 135.8 seconds, recorded decode 120.75 tokens/sec. craftbook-code-review passed 4 deterministic craftbook checks\n- [craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/facts.json): pass, fixed 9.5, 104.1 seconds, recorded decode 124.16 tokens/sec. craftbook-code-review passed 4 deterministic craftbook checks\n- [craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-57-00-486Z-9lvs/facts.json): pass, fixed 9.5, 114.7 seconds, recorded decode 117.15 tokens/sec. craftbook-code-review passed 4 deterministic craftbook checks\n\nThese historical runs differ in model, native and/or source identity and are not a controlled performance comparison. Earlier pass count: 3/3. Keep frozen scores separate from the advisory content review above.\n" }, { "cellId": "r1-saluki-cuda-developer-craftbook-api-contract-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-api-contract-review", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T16:46:58.461Z", "finishedAt": "2026-11-04T17:42:07.995Z", "durationMs": 3309534, "success": true, "reason": "craftbook-api-contract-review passed 6 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 365617 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-api-contract-review", "bytes": 16411, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.8834616713177285 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 23 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 365617 }, { "source": "facts", "path": "outcome.durationMs", "value": 3309534 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" }, "facts": { "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 553, "peakRssMb": 14937, "peakCpuPercent": 1343.8, "firstAtMs": 1, "lastAtMs": 3296340 }, "gpu": { "available": true, "sampleCount": 553, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 553, "peakUsedMb": 31634, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 67995, "totalOutputTokens": 25894, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 67951, "todayTokensCached": 57244, "todayTokensOut": 19547, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 67951, "totalTokensCached": 57244, "totalTokensOut": 19547, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 6967, "cachedInputTokens": 6671, "outputTokens": 150, "cost": 0, "durationMs": 119010, "outputTokensPerSec": 8.526014099280685, "at": "2026-11-04T17:41:59.881Z" }, "lastUpdated": "2026-11-04T17:41:59.881Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T17:42:02.496Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3297772, "meanTokensPerSec": 8.46, "genTokensPerSec": 8.46, "genTokens": 25894, "promptTokensPerSec": 402.64, "peakRssMb": 14937, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 67951, "output": 19547 } } } }, "craftbook": { "craftbookId": "api-contract-review", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "api-contract-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T16:47:06.057Z", "completedAt": "2026-11-04T16:53:04.249Z", "ms": 358192, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "4dc4e148-af81-49a8-aae6-8c2a71ee9ebf" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "review", "activatedAt": "2026-11-04T16:53:04.250Z", "completedAt": "2026-11-04T17:10:53.042Z", "ms": 1068792, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "70448446-28b8-4446-a368-a776df72ba79" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-04T17:10:53.044Z", "completedAt": "2026-11-04T17:31:49.394Z", "ms": 1256350, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "fb7d968c-a02d-4ec4-894a-9a5f7a41e902" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T17:31:49.396Z", "completedAt": "2026-11-04T17:39:56.512Z", "ms": 487116, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "fb7d968c-a02d-4ec4-894a-9a5f7a41e902" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T17:39:56.513Z", "completedAt": "2026-11-04T17:41:59.837Z", "ms": 123324, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "481df8d2-defd-41f0-b6cf-5af969277c73" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T16:47:04.288Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T16:47:04.203Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 487116 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "mahua": 2, "shreya": 2 }, "perTask": { "api-contract-review-workflow-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.2191009521484375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2191009521484375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-api-contract-review passed 6 deterministic craftbook checks", "durationMs": 3309534, "timeoutMs": 3746098, "budgetUsedFraction": 0.8834616713177285 }, "timing": { "startedAt": "2026-11-04T16:46:58.461Z", "finishedAt": "2026-11-04T17:42:07.995Z", "timeToFirstArtifactMs": 365617, "timeToLastArtifactWriteMs": 3177971, "timeToFirstTokenMs": 33751, "firstTurnTtftMs": 20223, "timeToFirstToolCallMs": 64971 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 23, "byTool": { "read_file": 7, "write_artifact": 5, "read_task_notes": 1, "read_files": 1, "read_artifact": 5, "stat": 1, "validate": 1, "write_task_note": 1, "advance_task_step": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 11 }, "sniff": { "progression": [ { "atMs": 7844, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, openapi.yaml, src/routes.ts, src/session-routes.ts, docs/api-contract.md" }, { "atMs": 66253, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 2, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 368470, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 1440303, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 4, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 2696612, "filePath": "craftbook-api-contract-review", "bytes": 16411, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 3303530, "filePath": "craftbook-api-contract-review", "bytes": 16411, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-api-contract-review", "bytes": 16411, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T16:46:58.464Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "[2026-11-04T16:46:58.464Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-api-contract-review-olAGXz", "[2026-11-04T16:46:58.464Z] [trial] id=craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2 scenario=craftbook-api-contract-review model=underdog-saluki-27b-iq2-mix", "[2026-11-04T16:46:58.464Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T16:46:58.466Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T16:46:58.464Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T16:46:58.471Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T16:46:58.476Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-api-contract-review-olAGXz", "[2026-11-04T16:46:58.477Z] [trial] throughput-scaled ceiling: 30m → 62m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T16:47:04.206Z] [trial] daemon spawned pid=37764 port=https://127.0.0.1:54847", "[2026-11-04T16:47:05.628Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T16:47:05.645Z] [trial] running scenario setup hook", "[2026-11-04T16:47:05.645Z] [trial] meester=sulien", "[2026-11-04T16:47:06.214Z] [trial] scenario setup complete", "[2026-11-04T16:47:06.215Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T16:47:06.267Z] [poll] initial digests hard=9f0da22827121588 soft=407b6b87469303c7", "[2026-11-04T16:47:06.347Z] [poll] hard-progress: 9f0da22827121588 → b45728ab7b7fd063 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T16:47:11.466Z] [poll] hard-progress: b45728ab7b7fd063 → 557116003d87a46e after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T16:48:04.789Z] [poll] hard-progress: 557116003d87a46e → fac3a77938ff58be after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=85,stream=236) sniff=2/0B)", "[2026-11-04T16:53:07.000Z] [poll] hard-progress: fac3a77938ff58be → 299fcc19f21c38cf after 302s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=931,stream=4116) sniff=3/0B)", "[2026-11-04T16:53:48.596Z] [poll] hard-progress: 299fcc19f21c38cf → efa79865ce647b55 after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=974,stream=4257) sniff=3/0B)", "[2026-11-04T17:10:53.041Z] [poll] hard-progress: efa79865ce647b55 → 93b40ca800fe110e after 1024s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=3761,stream=16427) sniff=3/0B)", "[2026-11-04T17:10:58.808Z] [poll] hard-progress: 93b40ca800fe110e → 7020f2070fc3bca1 after 6s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=9,slot=3761,stream=16427) sniff=4/0B)", "[2026-11-04T17:11:40.698Z] [poll] hard-progress: 7020f2070fc3bca1 → aac001f0ead49340 after 42s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=11,slot=3789,stream=16487) sniff=4/0B)", "[2026-11-04T17:12:16.123Z] [poll] hard-progress: aac001f0ead49340 → bea15c39cf2b58e3 after 35s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=12,slot=3859,stream=16685) sniff=4/0B)" ] }, "metrics": { "process": { "sampleCount": 553, "peakRssMb": 14937, "peakCpuPercent": 1343.8, "firstAtMs": 1, "lastAtMs": 3296340 }, "gpu": { "available": true, "sampleCount": 553, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 553, "peakUsedMb": 31634, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 67995, "totalOutputTokens": 25894, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 67951, "todayTokensCached": 57244, "todayTokensOut": 19547, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 67951, "totalTokensCached": 57244, "totalTokensOut": 19547, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 6967, "cachedInputTokens": 6671, "outputTokens": 150, "cost": 0, "durationMs": 119010, "outputTokensPerSec": 8.526014099280685, "at": "2026-11-04T17:41:59.881Z" }, "lastUpdated": "2026-11-04T17:41:59.881Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T17:42:02.496Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3297772, "meanTokensPerSec": 8.46, "genTokensPerSec": 8.46, "genTokens": 25894, "promptTokensPerSec": 402.64, "peakRssMb": 14937, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 67951, "output": 19547 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T16:46:58.461Z", "finishedAt": "2026-11-04T17:42:07.995Z", "timeToFirstArtifactMs": 365617, "timeToLastArtifactWriteMs": 3177971, "timeToFirstTokenMs": 33751, "firstTurnTtftMs": 20223, "timeToFirstToolCallMs": 64971 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "api-contract-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-04T16:47:06.057Z", "completedAt": "2026-11-04T16:53:04.249Z", "ms": 358192, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "4dc4e148-af81-49a8-aae6-8c2a71ee9ebf" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "review", "activatedAt": "2026-11-04T16:53:04.250Z", "completedAt": "2026-11-04T17:10:53.042Z", "ms": 1068792, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "70448446-28b8-4446-a368-a776df72ba79" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-04T17:10:53.044Z", "completedAt": "2026-11-04T17:31:49.394Z", "ms": 1256350, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "fb7d968c-a02d-4ec4-894a-9a5f7a41e902" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-04T17:31:49.396Z", "completedAt": "2026-11-04T17:39:56.512Z", "ms": 487116, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "fb7d968c-a02d-4ec4-894a-9a5f7a41e902" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "finish", "activatedAt": "2026-11-04T17:39:56.513Z", "completedAt": "2026-11-04T17:41:59.837Z", "ms": 123324, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "481df8d2-defd-41f0-b6cf-5af969277c73" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T16:47:04.288Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T16:47:04.203Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 487116 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "mahua": 2, "shreya": 2 }, "perTask": { "api-contract-review-workflow-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.2191009521484375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2191009521484375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "scenarioId": "craftbook-api-contract-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "artifactBytes": 16510, "artifactSha256": "9e2447bb4db000c67544de820b409f26448ce4664d893c9d58993e4010b781f9", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/ap/api-contract-review/versions/1.1.5/test.json", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": { "workflow fidelity": 8, "usefulness": 5 }, "meanScore": 6.5, "justification": "The complete workflow produces prioritized findings and concrete snippets, correctly finding an undocumented route and the retryCount mismatch. It misses the spec-only history route and assumes no existing consumers merely because no prior spec is supplied. The final reviewer checks report structure but misses that omission.", "reviewBasis": "Read the complete final report, supplied OpenAPI, both route modules and conventions; compared to the earlier reviewed scope/intermediate artifacts and read the final verdict. The final report reconciles the earlier WidgetCreate/displayName inconsistency. All judgments are against offline fixtures, not a deployed API.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "bytes": 5359, "sha256": "8a7692426d1300ac1d5175ab0ccd91e0fcdee5d15c4d3b032e119b5f2bec4453", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "bytes": 16154, "sha256": "e695bd189893d9b14a94f466dbccbe79975386582916dcd9ac7a709ce0862e47", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/artifacts/api-contract-review-workflow-eval/tasks/eval/verdict.md", "bytes": 1370, "sha256": "f76646b3c33709329145ca269f6c5c5e474f81facccd5438402d3a7534ac810d", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:54.769Z", "reviewId": "029", "cellId": "r1-saluki-cuda-developer-craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "scenarioId": "craftbook-api-contract-review", "status": "scored", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "baselineCellId": "r1-saluki-cuda-developer-craftbook-api-contract-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "1a18d4c473ee81806c748dd2f3bc9d4d8a0cba09a216621ab588622e910897ee", "evidenceSha256": "66de4683d4e15fef9b55d289dd77584eb41641eac3e6d0ce1a1b1b4172660027", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "bytes": 16510, "sha256": "9e2447bb4db000c67544de820b409f26448ce4664d893c9d58993e4010b781f9", "shownBytes": 16510, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2\\workspace\\api-contract-review-workflow-eval\\source\\craftbook-eval-brief.md", "bytes": 2833, "sha256": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "shownBytes": 2833, "truncated": false } ], "scoreAxes": { "workflow fidelity": 7, "usefulness": 8 }, "justification": "The final report uses the locked C1–C8 checklist, cites scope.md, and gives a verdict, severity-tiered findings, a compat verdict, and corrected YAML for each issue. Weaknesses: one \"corrected\" snippet still shows the wrong '200'. The strengths section says the router uses :widgetId while the finding quotes /widgets/:id.", "limitations": "Only the final api-review.md is shown. scope.md, review.md, gate results and the actual openapi.yaml are not visible, so spec-anchoring and phase handoffs can't be checked. The assigned step was scope, but the evidence is the last-phase output.", "meanScore": 7.5, "durationMs": 17347, "costUsd": 0.0473258, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 10056, "cache_read_input_tokens": 778, "output_tokens": 702, "output_tokens_details": { "thinking_tokens": 357 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 10056, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 702, "cache_read_input_tokens": 778, "cache_creation_input_tokens": 10056, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 10056 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 702, "cacheReadInputTokens": 778, "cacheCreationInputTokens": 10056, "webSearchRequests": 0, "costUSD": 0.0473258, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 357, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 10056, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-api-contract-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 55m 10s; budget used: 88.3%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 365617`; `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":16411,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.8834616713177285`; `facts.toolUse.totalToolCalls: 23`; `facts.timing.timeToFirstArtifactMs: 365617`; `facts.outcome.durationMs: 3309534` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 5 + 0.15 * 10 = **9.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14937 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31634 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.46 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 67995 + 25894 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 487s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 22% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 365617`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":16411,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.8834616713177285`\n- `facts.toolUse.totalToolCalls: 23`\n- `facts.timing.timeToFirstArtifactMs: 365617`\n- `facts.outcome.durationMs: 3309534`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 365617`; `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":16411,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.8834616713177285`; `facts.toolUse.totalToolCalls: 23`; `facts.timing.timeToFirstArtifactMs: 365617`; `facts.outcome.durationMs: 3309534`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nRead the complete final report, supplied OpenAPI, both route modules and conventions; compared to the earlier reviewed scope/intermediate artifacts and read the final verdict. The final report reconciles the earlier WidgetCreate/displayName inconsistency. All judgments are against offline fixtures, not a deployed API.\n\nThe final primary artifact is 16510 UTF-8 bytes; sniff.latest.bytes records 16411. Exact declared axes:\n\n| Axis | Score | Description |\n|---|---:|---|\n| workflow fidelity | 8 | The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute. |\n| usefulness | 5 | The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief. |\n\n**Advisory mean: 6.5/10, codex-inline/current-session.** Fixed composite remains **9/10** and outcome remains **pass**.\n\nThe complete workflow produces prioritized findings and concrete snippets, correctly finding an undocumented route and the retryCount mismatch. It misses the spec-only history route and assumes no existing consumers merely because no prior spec is supplied. The final reviewer checks report structure but misses that omission.\n\n## Positives\n\n- All six deterministic checks passed, with the real attributed workflow reaching terminal completion (outcome.success; sniff.latest.score: 6/6; craftbook.reachedTerminal). The scope, intermediate review, final report and verdict are all present; each primary artifact write completed (toolUse.byTool.write_artifact: 5 includes the rejected verdict draft).\n- The final report correctly identifies the implemented but undocumented single-widget GET and the retryCount string-versus-number mismatch. It recognizes the imported sessions handler and correctly accepts omission of the optional nullable owner email (artifacts.otherFileCount; captured source files and final report). The final snippets fix the intermediate draft’s inconsistent display_name/displayName naming.\n\n## Negatives\n\n- The report never identifies GET /widgets/{widgetId}/history as declared but unimplemented in either route module; its only criticism of that route concerns error responses (sniff.latest.score: 6/6 is a workflow verdict; advisory-artifact-review.json records the substantive omission). The earlier scope already lists both inventories, so the evidence was available.\n- “Backward-compatible: yes” and “no shipped contract is broken” assume that the absence of a prior spec proves the absence of consumers. Proposed path/field renames and array-to-envelope changes would affect consumers of the supplied interface; deployment status is not established by the fixtures (final artifact; source/craftbook-eval-brief.md). Compatibility should be conditional on the unverified pre-release assumption.\n- The report asserts that Revision and Session fields are always present without a supporting implementation, and its corrected create-response snippet retains a 200 response labeled wrong alongside the proposed 201. The reviewer verdict marks coverage and compatibility PASS based on the report rather than checking those claims against the source inventories (toolUse.byTool.read_artifact: 5; tasks/eval/verdict.md).\n- Generation consumed 25894 tokens and 55m10s for a 16510-byte final report plus two substantial preparatory documents (perf.derived.genTokens; outcome.durationMs). A verdict write was refused at 17:37:20.646Z over the paraphrased quote “Breaking changes: None”, then saved at 17:39:56.432Z; the main report was already complete (advisory-tool-evidence.json).\n\n## Tactical experiment — not applied\n\nIn a future version of ../gilde/data/gezel-templates/re/reviewer/versions/1.4.1/about.md, add: “Compare inventories in both directions: declared-but-unimplemented and implemented-but-undeclared. Treat an absent prior version as unknown release history, not proof that consumers do not exist; state compatibility assumptions.” Test on API reviews and an unrelated configuration/schema migration review. Keep scenario names, exact endpoints and expected answers out of the guidance.\n\n## Strategic experiment — not applied\n\nPrototype a read-only inventory comparison helper under packages/script-stdlib/src that emits normalized declared and implemented operation sets plus unmatched members, with source citations across imported route modules. Feed those facts into the existing review step and require human/model interpretation of dynamic routes rather than pretending static extraction is complete. Separately, keep source-backed findings and design proposals distinct in the review schema. Measure missing-operation recall and false positives across multiple frameworks before adoption.\n\n## Performance and fit\n\nElapsed 55.16 minutes; 25894 recorded generated tokens; decode 8.46 tokens/sec, prefill 402.64 tokens/sec. Peak VRAM 10644/12227 MiB, process RSS 14937 MiB, whole-system RAM 31634/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). One native 0.1.57 CUDA launch, 65536 context, f16 KV, one slot, 4096 reasoning budget. Planner retains 39/64 dense FFN layers on GPU and 25 on CPU. This is mixed residency, not full GPU residency. Shared GPU memory and paging are not separately measured. No explicit CUDA OOM, context overflow, endpoint failure or unexpected native exit was found; stopped-turn messages accompany normal final teardown.\n\n## Generalization and next experiment\n\nThe prior code-review found all four seeded defect classes, whereas this API review missed one direction of contract drift. Both passed their fixed workflow gates, so those passes establish execution/completeness, not exhaustive review coverage. The API verdict also resembles the bibliography/A-B reviewers in accepting the author’s unsupported assurances. A reusable source-to-claim check is the appropriate follow-up experiment.\n\nNo tuning, prompts, gates, source files or model weights were changed. The frozen campaign remains the baseline for any later experiment.\n\n## Last three same-scenario trials\n\n- [craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/facts.json): pass, fixed 8.5, 840.0 seconds, recorded decode 118.59 tokens/sec. craftbook-api-contract-review passed 6 deterministic craftbook checks\n- [craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/facts.json): pass, fixed 8.5, 302.3 seconds, recorded decode 121.8 tokens/sec. craftbook-api-contract-review passed 6 deterministic craftbook checks\n- [craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-03-01-925Z-akke/facts.json): pass, fixed 8.5, 275.7 seconds, recorded decode 117.16 tokens/sec. craftbook-api-contract-review passed 6 deterministic craftbook checks\n\nEarlier outcomes: 3/3 passed. Model, native and/or source identities differ, so these are historical context rather than a controlled native-version comparison.\n" }, { "cellId": "r1-saluki-cuda-developer-craftbook-deep-security-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-deep-security-review", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T17:45:22.804Z", "finishedAt": "2026-11-04T18:04:54.245Z", "durationMs": 1171441, "success": false, "reason": "craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (4142 bytes) for 101s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 7/11: security-review/REPORT.md is missing required content: real file and line — nothing in its 4142 bytes matches /src/admin\\.ts:[0-9]+/. Add that content.", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (4142 bytes) for 101s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 7/11: security-review/REPORT.md is missing required content: real file and line — nothing in its 4142 bytes matches /src/admin\\.ts:[0-9]+/. Add that content." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 271860 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 4142, "score": 7, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content: real file and line — nothing in its 4142 bytes matches /src/admin\\.ts:[0-9]+/. Add that content." } } ] }, "quality": { "score": 6.4, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 63.6% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 271860 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 4142, "score": 7, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content: real file and line — nothing in its 4142 bytes matches /src/admin\\.ts:[0-9]+/. Add that content." } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.20847310098845964 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 23 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 271860 }, { "source": "facts", "path": "outcome.durationMs", "value": 1171441 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.3, "band": "needs-tuning" }, "facts": { "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 195, "peakRssMb": 14946, "peakCpuPercent": 1239.9, "firstAtMs": 0, "lastAtMs": 1155815 }, "gpu": { "available": true, "sampleCount": 195, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 195, "peakUsedMb": 31538, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 43940, "totalOutputTokens": 8931, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 22898, "todayTokensCached": 16908, "todayTokensOut": 2006, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 22898, "totalTokensCached": 16908, "totalTokensOut": 2006, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 17047, "cachedInputTokens": 16908, "outputTokens": 663, "cost": 0, "durationMs": 1150464, "outputTokensPerSec": 8.76267611440227, "at": "2026-11-04T18:04:44.937Z" }, "lastUpdated": "2026-11-04T18:04:44.937Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T18:04:46.380Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1157257, "meanTokensPerSec": 8.95, "genTokensPerSec": 8.95, "genTokens": 8931, "promptTokensPerSec": 416.04, "peakRssMb": 14946, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22898, "output": 2006 } } } }, "craftbook": { "craftbookId": "deep-security-review", "tasksObserved": 2, "craftbookTasks": 0, "reachedTerminal": false, "verdict": "artifact-only" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T17:45:28.796Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T17:45:28.698Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "erzsebet": 1, "rae": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 5, "maxContextFill": 0.1053466796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1053466796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (4142 bytes) for 101s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 7/11: security-review/REPORT.md is missing required content: real file and line — nothing in its 4142 bytes matches /src/admin\\.ts:[0-9]+/. Add that content.", "durationMs": 1171441, "timeoutMs": 5619147, "budgetUsedFraction": 0.20847310098845964 }, "timing": { "startedAt": "2026-11-04T17:45:22.804Z", "finishedAt": "2026-11-04T18:04:54.245Z", "timeToFirstArtifactMs": 271860, "timeToLastArtifactWriteMs": 1162133, "timeToFirstTokenMs": 32493, "firstTurnTtftMs": 20825, "timeToFirstToolCallMs": 47397 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 23, "byTool": { "read_file": 11, "list_dir": 2, "search": 1, "read_document": 1, "write_file": 5, "validate": 2, "stat": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 7947, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 0, "scoreMax": 11, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: src/admin.ts, src/profile.ts, src/reports.ts, docs/security.md" }, { "atMs": 48527, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 0, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 273021, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 4, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 667540, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 3, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 894279, "filePath": "craftbook-deep-security-review", "bytes": 5833, "score": 9, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content /Verified Safe|Not Statically Verifiable/ — nothing in its 5833 bytes matches. Add that content." }, { "atMs": 1061760, "filePath": "craftbook-deep-security-review", "bytes": 4142, "score": 7, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content: real file and line — nothing in its 4142 bytes matches /src/admin\\.ts:[0-9]+/. Add that content." } ], "latest": { "filePath": "craftbook-deep-security-review", "bytes": 4142, "score": 7, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content: real file and line — nothing in its 4142 bytes matches /src/admin\\.ts:[0-9]+/. Add that content." } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T17:45:22.808Z] [trial] id=craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9 scenario=craftbook-deep-security-review model=underdog-saluki-27b-iq2-mix", "[2026-11-04T17:45:22.808Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-deep-security-review-7VNY8q", "[2026-11-04T17:45:22.808Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T17:45:22.808Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "[2026-11-04T17:45:22.808Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T17:45:22.811Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T17:45:22.814Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T17:45:22.820Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-deep-security-review-7VNY8q", "[2026-11-04T17:45:22.820Z] [trial] throughput-scaled ceiling: 45m → 94m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T17:45:28.761Z] [trial] daemon spawned pid=32072 port=https://127.0.0.1:55815", "[2026-11-04T17:45:29.972Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T17:45:29.986Z] [trial] meester=dagny", "[2026-11-04T17:45:29.986Z] [trial] running scenario setup hook", "[2026-11-04T17:45:30.456Z] [trial] scenario setup complete", "[2026-11-04T17:45:30.457Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T17:45:30.531Z] [poll] initial digests hard=385237158085ef92 soft=36d2383ca6e0a053", "[2026-11-04T17:45:30.806Z] [poll] hard-progress: 385237158085ef92 → b92b4088bdb69c3b after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T17:46:11.379Z] [poll] hard-progress: b92b4088bdb69c3b → 06c347f0fab498e3 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=39,stream=84) sniff=0/0B)", "[2026-11-04T17:46:23.274Z] [poll] hard-progress: 06c347f0fab498e3 → efa657adb9fb949a after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=59,stream=141) sniff=0/0B)", "[2026-11-04T17:46:52.827Z] [poll] hard-progress: efa657adb9fb949a → 1848a56953f38e06 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=126,stream=341) sniff=0/0B)", "[2026-11-04T17:47:04.625Z] [poll] hard-progress: 1848a56953f38e06 → 1b1a4adbbfb1f012 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=152,stream=429) sniff=0/0B)", "[2026-11-04T17:47:46.224Z] [poll] hard-progress: 1b1a4adbbfb1f012 → c7f27e9a8a81d2fe after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=8,slot=277,stream=803) sniff=0/0B)", "[2026-11-04T17:49:55.886Z] [poll] hard-progress: c7f27e9a8a81d2fe → 629691afc661b553 after 130s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=648,stream=2243) sniff=4/0B)", "[2026-11-04T17:50:19.757Z] [poll] hard-progress: 629691afc661b553 → 3ad7e10da0c76b4d after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=11,slot=667,stream=2290) sniff=4/0B)", "[2026-11-04T17:50:43.682Z] [poll] poisoned-session recovery: erzsebet/025c9d96 last turn aborted; sending one direct repair turn" ] }, "metrics": { "process": { "sampleCount": 195, "peakRssMb": 14946, "peakCpuPercent": 1239.9, "firstAtMs": 0, "lastAtMs": 1155815 }, "gpu": { "available": true, "sampleCount": 195, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 195, "peakUsedMb": 31538, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 43940, "totalOutputTokens": 8931, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 22898, "todayTokensCached": 16908, "todayTokensOut": 2006, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 22898, "totalTokensCached": 16908, "totalTokensOut": 2006, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 17047, "cachedInputTokens": 16908, "outputTokens": 663, "cost": 0, "durationMs": 1150464, "outputTokensPerSec": 8.76267611440227, "at": "2026-11-04T18:04:44.937Z" }, "lastUpdated": "2026-11-04T18:04:44.937Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T18:04:46.380Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1157257, "meanTokensPerSec": 8.95, "genTokensPerSec": 8.95, "genTokens": 8931, "promptTokensPerSec": 416.04, "peakRssMb": 14946, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 22898, "output": 2006 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T17:45:22.804Z", "finishedAt": "2026-11-04T18:04:54.245Z", "timeToFirstArtifactMs": 271860, "timeToLastArtifactWriteMs": 1162133, "timeToFirstTokenMs": 32493, "firstTurnTtftMs": 20825, "timeToFirstToolCallMs": 47397 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T17:45:28.796Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T17:45:28.698Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "erzsebet": 1, "rae": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 5, "maxContextFill": 0.1053466796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1053466796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "scenarioId": "craftbook-deep-security-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/workspace/security-maintenance-eval/security-review/REPORT.md", "artifactBytes": 4156, "artifactSha256": "a1e841cba3902981dc32d44da473a6b832d9c29f418e385bc8d3b7106a58b469", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/de/deep-security-review/versions/1.1.5/test.json", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ], "scoreAxes": { "verification": 5, "systemic analysis": 3, "remediation": 4 }, "meanScore": 4, "justification": "The final report traces the three real admin flaws and offers concrete fixes, but JSON additionally flags two documented safe constructs that the prose clears. Systemic analysis regressed during a second worker’s overwrite, and required citation/section forms remain missing. The report and machine-readable findings are inconsistent.", "reviewBasis": "Read the final report and findings JSON against all three source modules and docs/security.md; inspected every captured write attempt, actor identity, repair nudge and validator result. Scores apply to the final report with its linked machine-readable deliverable, not the stronger earlier draft. No external scan, source edit or model retry was performed.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/workspace/security-maintenance-eval/security-review/findings.json", "bytes": 5079, "sha256": "d7cb739313d1545a8bb18586d2054ba75b656671f116e47f412098fbe578406d", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:00.204Z", "reviewId": "030", "cellId": "r1-saluki-cuda-developer-craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "scenarioId": "craftbook-deep-security-review", "status": "scored", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "baselineCellId": "r1-saluki-cuda-developer-craftbook-deep-security-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "0882cb4ad8676329759838add8e5e0ab2d8f8831b0617cb0b7bffc0882830b7d", "evidenceSha256": "a102016f36a43b172e99bab6063d903d7d4c9fc4650c91b012423572c65d956e", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/workspace/security-maintenance-eval/security-review/REPORT.md", "bytes": 4156, "sha256": "a1e841cba3902981dc32d44da473a6b832d9c29f418e385bc8d3b7106a58b469", "shownBytes": 4156, "truncated": false } ], "contextFiles": [], "scoreAxes": { "verification": 6, "systemic analysis": 5, "remediation": 6 }, "justification": "Report confirms the admin command and SQL injection paths with line cites, and correctly clears the ALLOWED_SORTS and fixed-exec decoys. It notes that /reports/rotate gives privileged ops to any user but doesn't file it as a finding. Fixes are concrete code. Grouping is only light, by module.", "limitations": "findings.json and the source files weren't shown, so JSON schema, theme fields, line accuracy and any missed leads (e.g. /reports/users, profile) can't be verified. Scores rest on REPORT.md alone.", "meanScore": 5.666666666666667, "durationMs": 17311, "costUsd": 0.024974, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4380, "cache_read_input_tokens": 0, "output_tokens": 745, "output_tokens_details": { "thinking_tokens": 424 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4380, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 745, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 4380, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4380 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 745, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 4380, "webSearchRequests": 0, "costUSD": 0.024974, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 424, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4380, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-deep-security-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 5.3 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 19m 31s; budget used: 20.8%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (4142 bytes) for 101s — a re-emission of identical bytes counts as un...`; `facts.timing.timeToFirstArtifactMs: 271860`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":4142,\"score\":7,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content: real file and...` |\n| Output quality (25%) | 6.4 / 10 | The latest scenario gate reached 63.6% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 271860`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":4142,\"score\":7,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content: real file and...` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.20847310098845964`; `facts.toolUse.totalToolCalls: 23`; `facts.timing.timeToFirstArtifactMs: 271860`; `facts.outcome.durationMs: 1171441` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 6.4 + 0.2 * 5 + 0.15 * 10 = **5.3**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14946 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31538 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.95 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 43940 + 8931 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 11% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (4142 bytes) for 101s — a re-emission of identical bytes counts as un...`\n- `facts.timing.timeToFirstArtifactMs: 271860`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":4142,\"score\":7,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content: real file and...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 63.6% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 271860`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":4142,\"score\":7,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content: real file and...`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.20847310098845964`\n- `facts.toolUse.totalToolCalls: 23`\n- `facts.timing.timeToFirstArtifactMs: 271860`\n- `facts.outcome.durationMs: 1171441`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (4142 bytes) for 101s — a re-emission of identical bytes counts as un...`; `facts.timing.timeToFirstArtifactMs: 271860`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":4142,\"score\":7,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content: real file and...`\n- Quality: The latest scenario gate reached 63.6% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 271860`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":4142,\"score\":7,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content: real file and...`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.20847310098845964`; `facts.toolUse.totalToolCalls: 23`; `facts.timing.timeToFirstArtifactMs: 271860`; `facts.outcome.durationMs: 1171441`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nRead the final report and findings JSON against all three source modules and docs/security.md; inspected every captured write attempt, actor identity, repair nudge and validator result. Scores apply to the final report with its linked machine-readable deliverable, not the stronger earlier draft. No external scan, source edit or model retry was performed.\n\nThe final primary artifact is 4156 UTF-8 bytes; sniff.latest.bytes records 4142. Exact declared axes:\n\n| Axis | Score | Description |\n|---|---:|---|\n| verification | 5 | Raw leads are confirmed as real source-to-sink or authorization paths. |\n| systemic analysis | 3 | Findings are grouped by root cause and blast radius. |\n| remediation | 4 | The report and JSON contain concrete, severity-consistent fixes. |\n\n**Advisory mean: 4/10, codex-inline/current-session.** Fixed composite remains **5.3/10** and outcome remains **fail**.\n\nThe final report traces the three real admin flaws and offers concrete fixes, but JSON additionally flags two documented safe constructs that the prose clears. Systemic analysis regressed during a second worker’s overwrite, and required citation/section forms remain missing. The report and machine-readable findings are inconsistent.\n\n## Positives\n\n- The final report and JSON both retain the three seeded admin issues: request data reaching a shell command, request data interpolated into SQL, and missing requireAdmin on the admin routes (sniff.latest.score: 7/11; captured src/admin.ts and final artifacts). Proposed argument-array execution, input validation, bound SQL values and route middleware address those concrete paths.\n- The original Reviewer’s first JSON contained exactly the three admin findings; after it was saved, the standard checker’s remaining failures concerned only the missing report (toolUse.byTool.write_file; log.txt at 17:49:55Z). The original Reviewer’s later report explicitly covered root causes, blast radius, safe constructs and path:line citations. This shows useful analysis existed before subsequent overwrites.\n\n## Negatives\n\n- Final findings.json adds command-injection and SQL-injection findings against the reports module on speculative future-change grounds, despite docs/security.md explicitly describing those constructs as safe and the final report itself clearing them (toolUse.byTool.write_file: 5; final findings F-004/F-005; oracle PRECISION FAILURE in log.txt). These are actual review-precision and cross-file consistency failures, not just a heading mismatch.\n- The final report uses “src/admin.ts line 4” rather than the required colon citation and lacks the recognized systemic and safe/uncertain section labels (sniff.latest.failReason; score 7/11). Natural-language file/line references and a Positive observations section convey some equivalent meaning, so those specific checks are formatting-sensitive. The JSON precision failure independently prevents a pass.\n- A missing-deliverable nudge created Developer Erzsebet at 17:47:46.152Z despite Reviewer Rae already working. Erzsebet overwrote Rae’s three-item findings list at 17:56:24.693Z, adding the two false positives, then overwrote Rae’s stronger report at 18:03:01.309Z; the measured checks fell from 9/11 to 7/11 (team.rolesCreated includes Reviewer and Developer; advisory-write-timeline.json). This is concrete competing-writer interference.\n- The trial ended after 101 seconds without changing the final failing report following repair feedback; it did not exhaust the 94-minute ceiling (outcome.failureMode: model-stuck; budgetUsedFraction: 0.2085). Generic validate calls only checked nonempty files/JSON parsing and reported PASS while the precision oracle and content checks still failed (toolUse.byTool.validate: 2). Six read-only calls also caused a separate local turn abort before the automatic repair. Neither event is a native OOM.\n- The original report write was refused for “Express” and source line numbers 13/14 despite the numbered source read. The revised original report was eventually saved, then overwritten by the repair worker (advisory-tool-evidence.json; daemon.log grounding entries). Persisted sessions omit one unfinished turn, so the recorded tool events are the primary evidence.\n\n## Tactical experiment — not applied\n\nIn a future version of ../gilde/data/gezel-templates/re/reviewer/versions/1.4.1/about.md, add: “Before saving linked prose and findings records, reconcile their IDs and dispositions. Do not list a construct as a confirmed vulnerability when its explanation says the current path is safe; place future hardening suggestions separately.” Verify on security reviews and ordinary code-review JSON/report pairs, measuring contradictory records without suppressing evidenced issues.\n\n## Strategic experiment — not applied\n\nIn evals/src/sniff-feedback.ts, resolve the actual active deliverable owner before falling back to an ensured Developer: an active Reviewer producing the requested review is a valid owner. Carry artifact version/hash into any delegated repair and reject stale full-file overwrites or merge a narrowly scoped change. The observed two-session chronology supports this ownership experiment directly. Test on security, document and data-review tasks; do not alter the running campaign or re-run this capability failure. Also distinguish generic file syntax validation from scenario acceptance in surfaced receipts so a nonempty-file PASS cannot be mistaken for semantic approval.\n\n## Performance and fit\n\nElapsed 19.52 minutes; 8931 recorded generated tokens; decode 8.95 tokens/sec, prefill 416.04 tokens/sec. Peak VRAM 10630/12227 MiB, process RSS 14946 MiB, whole-system RAM 31538/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). One native 0.1.57 CUDA launch, 65536 context, f16 KV, one slot, 4096 reasoning budget. Planner retains 39/64 dense FFN layers on GPU and 25 on CPU. This is mixed residency, not full GPU residency. Shared GPU memory and paging are not separately measured. No explicit CUDA OOM, context overflow, endpoint failure or unexpected native exit was found; stopped-turn messages accompany normal final teardown.\n\n## Generalization and next experiment\n\nUnlike the prior code-review, the final security JSON fails precision on explicitly documented safe patterns. The evidence also shows a framework contribution: the initial Reviewer had the correct three findings before the automatically introduced repair Developer replaced them. This should be reported as a failed model-plus-runner trial with an observed ownership/repair problem, not as an unsupported model ceiling or a hardware-load failure.\n\nNo tuning, prompts, gates, source files or model weights were changed. The frozen campaign remains the baseline for any later experiment.\n\n## Last three same-scenario trials\n\n- [craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/facts.json): fail, fixed 6.3, 192.0 seconds, recorded decode 132.51 tokens/sec. repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"\n- [craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/facts.json): fail, fixed 6.3, 175.3 seconds, recorded decode 131.6 tokens/sec. repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"\n- [craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-26-46-143Z-kmzn/facts.json): fail, fixed 6.3, 147.1 seconds, recorded decode 134.83 tokens/sec. repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 9 while the failing detail kept changing; latest checked file security-review/findings.json. Last failure: security-review/findings.json: record 0 is missing required field \"theme\"\n\nEarlier outcomes: 0/3 passed. Model, native and/or source identities differ, so these are historical context rather than a controlled native-version comparison.\n" }, { "cellId": "r1-saluki-cuda-developer-large-pr-review", "baselineCellId": "r1-saluki-cuda-developer-large-pr-review", "suite": "developer", "scenario": "large-pr-review", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T18:07:39.733Z", "finishedAt": "2026-11-04T19:34:22.018Z", "durationMs": 5202285, "success": true, "reason": "complete large-PR review (coverage-120-of-120, coverage-summary, late-defect-citation, request-changes, no-false-missing-api, source-unchanged)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2903642 }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6943605052510639 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 90 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2903642 }, { "source": "facts", "path": "outcome.durationMs", "value": 5202285 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 871, "peakRssMb": 14403, "peakCpuPercent": 1210.3, "firstAtMs": 1, "lastAtMs": 5187635 }, "gpu": { "available": true, "sampleCount": 871, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 871, "peakUsedMb": 31289, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 188878, "totalOutputTokens": 35283, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 31296, "todayTokensCached": 0, "todayTokensOut": 882, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 31296, "totalTokensCached": 0, "totalTokensOut": 882, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 31296, "outputTokens": 882, "cost": 0, "durationMs": 904103, "outputTokensPerSec": 8.473599049818116, "at": "2026-11-04T19:11:10.136Z" }, "lastUpdated": "2026-11-04T19:11:10.136Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T19:34:15.183Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5189126, "meanTokensPerSec": 8.27, "genTokensPerSec": 8.27, "genTokens": 35283, "promptTokensPerSec": 322.61, "peakRssMb": 14403, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 31296, "output": 882 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T18:07:45.667Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T18:07:45.584Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "rina": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.38409423828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.38409423828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 2 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "complete large-PR review (coverage-120-of-120, coverage-summary, late-defect-citation, request-changes, no-false-missing-api, source-unchanged)", "durationMs": 5202285, "timeoutMs": 7492196, "budgetUsedFraction": 0.6943605052510639 }, "timing": { "startedAt": "2026-11-04T18:07:39.733Z", "finishedAt": "2026-11-04T19:34:22.018Z", "timeToFirstArtifactMs": 2903642, "timeToLastArtifactWriteMs": 5191986, "timeToFirstTokenMs": 40959, "firstTurnTtftMs": 29265, "timeToFirstToolCallMs": 73434 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 90, "byTool": { "list_artifacts": 3, "read_artifact": 71, "list_dir": 5, "read_file": 4, "find_files": 2, "write_artifact": 2, "write_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 130 }, "sniff": { "progression": [], "latest": null }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T18:07:39.736Z] [trial] id=large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj scenario=large-pr-review model=underdog-saluki-27b-iq2-mix", "[2026-11-04T18:07:39.736Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-large-pr-review-l9k0Oj", "[2026-11-04T18:07:39.736Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T18:07:39.736Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "[2026-11-04T18:07:39.736Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T18:07:39.739Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T18:07:39.745Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T18:07:39.750Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-large-pr-review-l9k0Oj", "[2026-11-04T18:07:39.750Z] [trial] throughput-scaled ceiling: 60m → 125m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T18:07:45.562Z] [trial] daemon spawned pid=49204 port=https://127.0.0.1:57054", "[2026-11-04T18:07:46.975Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T18:07:46.992Z] [trial] meester=rosalind", "[2026-11-04T18:07:46.992Z] [trial] running scenario setup hook", "[2026-11-04T18:07:48.376Z] [trial] scenario setup complete", "[2026-11-04T18:07:48.378Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T18:07:48.457Z] [poll] initial digests hard=9dc0fd59becebac1 soft=e2a9636a58de3992", "[2026-11-04T18:07:48.551Z] [poll] hard-progress: 9dc0fd59becebac1 → 7e96a6bf6580c2d6 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-04T18:08:58.147Z] [poll] hard-progress: 7e96a6bf6580c2d6 → c16b8f4c347ef788 after 70s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=103,stream=269) sniff=0/0B)", "[2026-11-04T18:09:39.922Z] [poll] hard-progress: c16b8f4c347ef788 → 2539543f076b2851 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=144,stream=421) sniff=0/0B)", "[2026-11-04T18:11:14.793Z] [poll] hard-progress: 2539543f076b2851 → a102873394fca680 after 95s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=246,stream=709) sniff=0/0B)", "[2026-11-04T18:14:12.186Z] [poll] hard-progress: a102873394fca680 → ebb2499b853353a6 after 177s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=33,slot=626,stream=2563) sniff=0/0B)", "[2026-11-04T18:14:18.274Z] [poll] retry-loop nudge deferred: rina/5bca0287 in large-pr-review-eval for 390s, last tool call 6s ago is already working; refusing to queue a potentially stale edit", "[2026-11-04T18:15:29.813Z] [poll] hard-progress: ebb2499b853353a6 → deb78b71fcffc79d after 78s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=36,slot=695,stream=2784) sniff=0/0B)", "[2026-11-04T18:15:47.570Z] [poll] hard-progress: deb78b71fcffc79d → 6a4db0b517201658 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=37,slot=722,stream=2883) sniff=0/0B)", "[2026-11-04T18:16:11.655Z] [poll] hard-progress: 6a4db0b517201658 → 267339a5ee7001cf after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=38,slot=750,stream=2952) sniff=0/0B)" ] }, "metrics": { "process": { "sampleCount": 871, "peakRssMb": 14403, "peakCpuPercent": 1210.3, "firstAtMs": 1, "lastAtMs": 5187635 }, "gpu": { "available": true, "sampleCount": 871, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 871, "peakUsedMb": 31289, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 188878, "totalOutputTokens": 35283, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 31296, "todayTokensCached": 0, "todayTokensOut": 882, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 31296, "totalTokensCached": 0, "totalTokensOut": 882, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 31296, "outputTokens": 882, "cost": 0, "durationMs": 904103, "outputTokensPerSec": 8.473599049818116, "at": "2026-11-04T19:11:10.136Z" }, "lastUpdated": "2026-11-04T19:11:10.136Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T19:34:15.183Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5189126, "meanTokensPerSec": 8.27, "genTokensPerSec": 8.27, "genTokens": 35283, "promptTokensPerSec": 322.61, "peakRssMb": 14403, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 31296, "output": 882 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T18:07:39.733Z", "finishedAt": "2026-11-04T19:34:22.018Z", "timeToFirstArtifactMs": 2903642, "timeToLastArtifactWriteMs": 5191986, "timeToFirstTokenMs": 40959, "firstTurnTtftMs": 29265, "timeToFirstToolCallMs": 73434 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T18:07:45.667Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T18:07:45.584Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "rina": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.38409423828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.38409423828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 2 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-large-pr-review", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "scenarioId": "large-pr-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/workspace/large-pr-review-eval/pr-review.md", "artifactBytes": 3191, "artifactSha256": "1c71327ff54f70166e6a640f393e58c9f01c2d538ff8171f756cadc9b5b898ff", "axisSource": "evals/src/scenarios/large-pr-review.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared qualitative axes. The report identifies the real authorization defect and resolves the API definition, but its claim that all 120 patch records were read is unsupported: only 55 distinct record paths have successful read requests. The final 120-entry ledger establishes claimed coverage, not observed coverage.", "reviewBasis": "Read the final report and both coverage ledgers, matched successful patch-record requests to the supplied corpus, checked relevant source modules and the frozen acceptance logic. Rechecked the final recording after the provisional review; it reports zero capture gaps. No axes were invented.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/workspace/large-pr-review-eval/pr-review-coverage.json", "bytes": 11426, "sha256": "65e778c7577aa11ffb3fcf2f46c6a29ff568a25613c2293b15c6db1424a2e0ea", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/artifacts/large-pr-review-eval/pr-review-coverage.json", "bytes": 6027, "sha256": "824800cc7c6806e8588dd7df4bcd2221b8f5ee2af5cb776d7e1b4e36dea6d252", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:58:59.723Z", "reviewId": "031", "cellId": "r1-saluki-cuda-developer-large-pr-review", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "scenarioId": "large-pr-review", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "baselineCellId": "r1-saluki-cuda-developer-large-pr-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "5bafc3f9f2b50b431fac8d5091a0eaef063dfc667029a7564d010f37a76b73e0", "evidenceSha256": "1ccf6d6bbf6e3250b99f3d12ab919182469402ef96dc394b0e1b53ac0f3fa1ac", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/workspace/large-pr-review-eval/pr-review.md", "bytes": 3191, "sha256": "1c71327ff54f70166e6a640f393e58c9f01c2d538ff8171f756cadc9b5b898ff", "shownBytes": 3191, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The report has the required Coverage line, findings table, and request-changes verdict. It cites the authorization defect at src/security/late-authorization.ts:4 and resolves assuredApi to its later definition instead of calling it missing.", "limitations": "Only pr-review.md was shown. pr-review-coverage.json, batch sizes, tool use and the 120/120 claim can't be verified. The report's \"line 3\" comment and padding-file claims are unconfirmed. Source-only review.", "meanScore": null, "durationMs": 13917, "costUsd": 0.014606, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2923, "cache_read_input_tokens": 700, "output_tokens": 284, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2923, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 284, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2923, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2923 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 284, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2923, "webSearchRequests": 0, "costUSD": 0.014606, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2923, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - large-pr-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 86m 42s; budget used: 69.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 2903642`; `facts.sniff.latest: null` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.6943605052510639`; `facts.toolUse.totalToolCalls: 90`; `facts.timing.timeToFirstArtifactMs: 2903642`; `facts.outcome.durationMs: 5202285` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14403 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31289 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.27 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 188878 + 35283 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 38% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 2 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 2903642`\n- `facts.sniff.latest: null`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.6943605052510639`\n- `facts.toolUse.totalToolCalls: 90`\n- `facts.timing.timeToFirstArtifactMs: 2903642`\n- `facts.outcome.durationMs: 5202285`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 2903642`; `facts.sniff.latest: null`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.6943605052510639`; `facts.toolUse.totalToolCalls: 90`; `facts.timing.timeToFirstArtifactMs: 2903642`; `facts.outcome.durationMs: 5202285`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nRead the final report and both coverage ledgers, matched successful patch-record requests to the supplied corpus, checked relevant source modules and the frozen acceptance logic. Rechecked the final recording after the provisional review; it reports zero capture gaps. No axes were invented.\n\nPrimary artifact: 3191 UTF-8 bytes. Reviewer: codex-inline/current-session.\n\nNo declared qualitative axes; this is a prose-only advisory review.\n\nNo declared qualitative axes. The report identifies the real authorization defect and resolves the API definition, but its claim that all 120 patch records were read is unsupported: only 55 distinct record paths have successful read requests. The final 120-entry ledger establishes claimed coverage, not observed coverage.\n\nFixed composite remains **8.5/10**; outcome remains **pass**. This advisory review does not alter the deterministic score.\n\n## Positives\n\n- The fixed trial passed all six acceptance signals and left the checkout source unchanged (outcome.success; outcome.reason). The final workspace ledger names all 120 required file paths, and the report correctly asks for changes on the missing ownership check at src/security/late-authorization.ts:4.\n- The report verifies the late assuredApi definition instead of claiming the imported API is absent, satisfying the no-false-missing-api check (outcome.reason). The security finding and relevant code citations match the captured source.\n\n## Negatives\n\n- The report says every one of 120 records was read and verified against the checkout. The complete event trace has successful requests for only 55 distinct patch-record paths: 001–050, 069, 105, 111, 119 and 120. This is a count of requested record paths, not a guarantee that every line of even those 55 reached the model. The 120-item final self-authored ledger cannot substantiate the stronger claim (toolUse.byTool.read_artifact: 71; advisory-coverage-audit.json; recording has 0 gaps).\n- The final report was saved at 19:11:10.141Z; after one further flags-index read, the only remaining tool call wrote the 120-entry workspace ledger at 19:34:11.729Z. No additional patch review occurred before the pass. The earlier artifact-drawer ledger remains at 50 entries (toolUse.byTool.write_artifact: 2, write_file: 3; captured writes). Listing file names or a flags index is not equivalent to reading patches.\n- Several guessed opaque record names failed to resolve; the successful path inventory was already supplied in the corpus manifest. Ninety tool calls and 35283 generated tokens took 86m42s, including the late ledger write (toolUse.totalToolCalls; perf.derived.genTokens; outcome.durationMs). Informational cleared items are also numbered as findings, adding noise to the one actionable defect.\n\n## Tactical experiment — not applied\n\nFor a future version of ../gilde/data/gezel-templates/re/reviewer/versions/1.4.1/about.md, add: “Copy opaque record paths from the supplied inventory; never reconstruct their hashes. Keep listed, requested, actually returned, and reviewed items distinct. Report unfinished coverage explicitly instead of filling a completion ledger from the inventory.” Measure evidence-backed coverage and failed path lookups on large reviews and unrelated literature/document reviews. No live prompt change is applied.\n\n## Strategic experiment — not applied\n\nReuse packages/service/src/chat/artifact-read-evidence.ts to retain service-written read-slice receipts keyed by path, range and content hash, then expose aggregate observed coverage to review workflows through packages/service/src/chat/evidence-ledger.ts. Compare self-reported coverage to actual returned ranges and identify unreviewed records before allowing an unqualified full-coverage statement. The current evals/src/scenarios/large-pr-review.ts check trusts reviewedFiles; retain its frozen result here. This is a general evidence/provenance experiment, not a changed gate or retry.\n\n## Performance and fit\n\nElapsed 86.70 minutes; 35283 recorded generated tokens; mean decode 8.27 tokens/sec, prefill 322.61 tokens/sec. Peak VRAM 10642/12227 MiB, process RSS 14403 MiB, whole-system RAM 31289/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). One native 0.1.57 CUDA launch, 65536-token context, f16 KV, one slot, 4096 reasoning budget. The planner places dense FFN from 39/64 layers on GPU and 25 on CPU. This demonstrates mixed residency, not a fully GPU-resident 27B model. Shared GPU memory and paging are not separately measured. No explicit CUDA OOM, context overflow, endpoint failure or unexpected native exit was found in this trial.\n\n## Generalization and next experiment\n\nThe unrelated codemod sweep below has a real search receipt and a byte-for-byte mechanical transformation, so its coverage claim is substantially better supported. Bibliography and API-contract reviews also show why an artifact/workflow pass alone does not establish source-complete analysis. Compare observed-read completeness and claim accuracy across those task types before adopting a runtime change.\n\nThe concrete future experiment is to carry source/execution receipts across role handoffs, then measure evidence-backed completion claims and false corrections across code review, codemods and unrelated research tasks. No tuning, prompts, gates, weights or source code were changed in this frozen campaign.\n\n## Last three same-scenario trials\n\n- [large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/facts.json): pass, fixed 8.5, 1379.8 seconds, recorded decode 136.87 tokens/sec. complete large-PR review (coverage-120-of-120, coverage-summary, late-defect-citation, request-changes, no-false-missing-api, source-unchanged)\n- [large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i/facts.json): fail, fixed 3.3, 1349.6 seconds, recorded decode 134.89 tokens/sec. retry loop (fast-path): sniff \"large-pr-review:5:targetpr-review.md:fr1i0sit8:rp0:rf0\" stuck for 8m — 4 re-writes (6 tool calls) without sniff movement. Artifact produced but never reached success.\n- [large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-35-27-000Z-vetk/facts.json): pass, fixed 8.5, 810.8 seconds, recorded decode 135.16 tokens/sec. complete large-PR review (coverage-120-of-120, coverage-summary, late-defect-citation, request-changes, no-false-missing-api, source-unchanged)\n\nEarlier outcomes: 2/3 passed. Different model/native/source identities make these historical context, not a controlled native-version comparison.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json and historical-comparison.json beside this report; the specialized advisory audit preserves the coverage or fixture comparison.\n" }, { "cellId": "r1-saluki-cuda-developer-craftbook-codemod-sweep", "baselineCellId": "r1-saluki-cuda-developer-craftbook-codemod-sweep", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T19:37:25.767Z", "finishedAt": "2026-11-04T20:05:30.995Z", "durationMs": 1685228, "success": true, "reason": "craftbook-codemod-sweep passed 32 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 340691 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 32, "scoreMax": 32, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.14995409000281448 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 46 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 340691 }, { "source": "facts", "path": "outcome.durationMs", "value": 1685228 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 281, "peakRssMb": 15548, "peakCpuPercent": 1683.2, "firstAtMs": 0, "lastAtMs": 1669441 }, "gpu": { "available": true, "sampleCount": 281, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 281, "peakUsedMb": 31436, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 56535, "totalOutputTokens": 12807, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 56459, "todayTokensCached": 51722, "todayTokensOut": 2416, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 56459, "totalTokensCached": 51722, "totalTokensOut": 2416, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10426, "cachedInputTokens": 9813, "outputTokens": 81, "cost": 0, "durationMs": 142583, "outputTokensPerSec": 8.85884657153225, "at": "2026-11-04T20:05:20.609Z" }, "lastUpdated": "2026-11-04T20:05:20.609Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T20:05:22.950Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1671018, "meanTokensPerSec": 8.8, "genTokensPerSec": 8.8, "genTokens": 12807, "promptTokensPerSec": 392.26, "peakRssMb": 15548, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 56459, "output": 2416 } } } }, "craftbook": { "craftbookId": "codemod-sweep", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "profile-cards/1", "stepId": "enumerate", "activatedAt": "2026-11-04T19:37:33.208Z", "completedAt": "2026-11-04T19:43:06.628Z", "ms": 333420, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "f391a9f3-3e73-4a61-808a-026c21946b8b" ] }, { "ref": "profile-cards/1", "stepId": "apply", "activatedAt": "2026-11-04T19:43:06.629Z", "completedAt": "2026-11-04T19:46:14.449Z", "ms": 187820, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "b17110d3-fe60-4282-98b2-1cfc3abd9769" ] }, { "ref": "profile-cards/1", "stepId": "verify", "activatedAt": "2026-11-04T19:46:14.451Z", "completedAt": "2026-11-04T19:50:29.614Z", "ms": 255163, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "b17110d3-fe60-4282-98b2-1cfc3abd9769" ] }, { "ref": "profile-cards/1", "stepId": "evaluate", "activatedAt": "2026-11-04T19:50:29.615Z", "completedAt": "2026-11-04T20:02:52.976Z", "ms": 743361, "gateRejections": 0, "toolCalls": 13, "sessionIds": [ "e4217fb0-3191-4e86-b142-0a4edf6189ef" ] }, { "ref": "profile-cards/1", "stepId": "finish", "activatedAt": "2026-11-04T20:02:52.977Z", "completedAt": "2026-11-04T20:05:20.567Z", "ms": 147590, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "6f5cf3fd-3b2f-4a65-88de-2ce213d006e2" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T19:37:31.429Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T19:37:31.328Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 255163 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "agustina": 1, "ioana": 1, "nasser": 1, "vishal": 1 }, "perTask": { "profile-cards/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.129730224609375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.129730224609375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-codemod-sweep passed 32 deterministic craftbook checks", "durationMs": 1685228, "timeoutMs": 11238293, "budgetUsedFraction": 0.14995409000281448 }, "timing": { "startedAt": "2026-11-04T19:37:25.767Z", "finishedAt": "2026-11-04T20:05:30.995Z", "timeToFirstArtifactMs": 340691, "timeToLastArtifactWriteMs": 1527235, "timeToFirstTokenMs": 34246, "firstTurnTtftMs": 20938, "timeToFirstToolCallMs": 48402 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Codebase Analyst", "Developer", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 46, "byTool": { "read_task_notes": 2, "read_file": 8, "list_dir": 3, "grep_files": 1, "write_artifact": 5, "read_artifact": 6, "read_files": 4, "replace_in_file": 7, "run_package_script": 2, "read_artifacts": 1, "validate": 4, "advance_task_step": 2, "write_task_note": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 16 }, "sniff": { "progression": [ { "atMs": 8122, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 9, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/api.js, src/index.js, src/profile.js, tests/profile.test.mjs, CHANGELOG.md, source/rename-brief.md" }, { "atMs": 48594, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 9, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/api.js, src/index.js, src/profile.js, tests/profile.test.mjs" }, { "atMs": 107542, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 10, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 344137, "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 15, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 469349, "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 20, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 529387, "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 24, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 785562, "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 27, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 1514967, "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 29, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 1665175, "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 31, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 1676882, "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 32, "scoreMax": 32, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-codemod-sweep", "bytes": 1659, "score": 32, "scoreMax": 32, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T19:37:25.771Z] [trial] id=craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6 scenario=craftbook-codemod-sweep model=underdog-saluki-27b-iq2-mix", "[2026-11-04T19:37:25.771Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "[2026-11-04T19:37:25.771Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-codemod-sweep-OlpEoQ", "[2026-11-04T19:37:25.771Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T19:37:25.771Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T19:37:25.773Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T19:37:25.777Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T19:37:25.782Z] [trial] throughput-scaled ceiling: 90m → 187m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T19:37:25.782Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-codemod-sweep-OlpEoQ", "[2026-11-04T19:37:31.430Z] [trial] daemon spawned pid=38528 port=https://127.0.0.1:50755", "[2026-11-04T19:37:32.761Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-04T19:37:32.777Z] [trial] meester=malika", "[2026-11-04T19:37:32.777Z] [trial] running scenario setup hook", "[2026-11-04T19:37:33.372Z] [trial] scenario setup complete", "[2026-11-04T19:37:33.373Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T19:37:33.439Z] [poll] initial digests hard=dfca3cb99056c039 soft=4bb07210c0c03495", "[2026-11-04T19:37:33.940Z] [poll] hard-progress: dfca3cb99056c039 → 546f69ff14705909 after 1s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-04T19:37:39.342Z] [poll] hard-progress: 546f69ff14705909 → d6cb876373ddcadb after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-04T19:38:14.427Z] [poll] hard-progress: d6cb876373ddcadb → b60df50ee8721991 after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=33,stream=71) sniff=9/0B)", "[2026-11-04T19:38:32.187Z] [poll] hard-progress: b60df50ee8721991 → e17384751efde7da after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=53,stream=134) sniff=9/0B)", "[2026-11-04T19:39:13.370Z] [poll] hard-progress: e17384751efde7da → fc71cad6eacac8c9 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=137,stream=383) sniff=10/0B)", "[2026-11-04T19:41:52.842Z] [poll] hard-progress: fc71cad6eacac8c9 → 38c1125067421137 after 159s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=600,stream=2128) sniff=10/0B)", "[2026-11-04T19:43:09.955Z] [poll] hard-progress: 38c1125067421137 → dc2a3be7b2fb549c after 77s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=12,slot=802,stream=3149) sniff=15/1659B)", "[2026-11-04T19:43:45.777Z] [poll] hard-progress: dc2a3be7b2fb549c → a6b8f8abafcbc382 after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=13,slot=824,stream=3190) sniff=15/1659B)", "[2026-11-04T19:43:57.596Z] [poll] hard-progress: a6b8f8abafcbc382 → facb7b6d9462d288 after 12s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=14,slot=846,stream=3288) sniff=15/1659B)" ] }, "metrics": { "process": { "sampleCount": 281, "peakRssMb": 15548, "peakCpuPercent": 1683.2, "firstAtMs": 0, "lastAtMs": 1669441 }, "gpu": { "available": true, "sampleCount": 281, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 281, "peakUsedMb": 31436, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 56535, "totalOutputTokens": 12807, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 56459, "todayTokensCached": 51722, "todayTokensOut": 2416, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 56459, "totalTokensCached": 51722, "totalTokensOut": 2416, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10426, "cachedInputTokens": 9813, "outputTokens": 81, "cost": 0, "durationMs": 142583, "outputTokensPerSec": 8.85884657153225, "at": "2026-11-04T20:05:20.609Z" }, "lastUpdated": "2026-11-04T20:05:20.609Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T20:05:22.950Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1671018, "meanTokensPerSec": 8.8, "genTokensPerSec": 8.8, "genTokens": 12807, "promptTokensPerSec": 392.26, "peakRssMb": 15548, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 56459, "output": 2416 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T19:37:25.767Z", "finishedAt": "2026-11-04T20:05:30.995Z", "timeToFirstArtifactMs": 340691, "timeToLastArtifactWriteMs": 1527235, "timeToFirstTokenMs": 34246, "firstTurnTtftMs": 20938, "timeToFirstToolCallMs": 48402 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "profile-cards/1", "stepId": "enumerate", "activatedAt": "2026-11-04T19:37:33.208Z", "completedAt": "2026-11-04T19:43:06.628Z", "ms": 333420, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "f391a9f3-3e73-4a61-808a-026c21946b8b" ] }, { "ref": "profile-cards/1", "stepId": "apply", "activatedAt": "2026-11-04T19:43:06.629Z", "completedAt": "2026-11-04T19:46:14.449Z", "ms": 187820, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "b17110d3-fe60-4282-98b2-1cfc3abd9769" ] }, { "ref": "profile-cards/1", "stepId": "verify", "activatedAt": "2026-11-04T19:46:14.451Z", "completedAt": "2026-11-04T19:50:29.614Z", "ms": 255163, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "b17110d3-fe60-4282-98b2-1cfc3abd9769" ] }, { "ref": "profile-cards/1", "stepId": "evaluate", "activatedAt": "2026-11-04T19:50:29.615Z", "completedAt": "2026-11-04T20:02:52.976Z", "ms": 743361, "gateRejections": 0, "toolCalls": 13, "sessionIds": [ "e4217fb0-3191-4e86-b142-0a4edf6189ef" ] }, { "ref": "profile-cards/1", "stepId": "finish", "activatedAt": "2026-11-04T20:02:52.977Z", "completedAt": "2026-11-04T20:05:20.567Z", "ms": 147590, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "6f5cf3fd-3b2f-4a65-88de-2ce213d006e2" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T19:37:31.429Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T19:37:31.328Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 255163 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "agustina": 1, "ioana": 1, "nasser": 1, "vishal": 1 }, "perTask": { "profile-cards/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.129730224609375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.129730224609375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "scenarioId": "craftbook-codemod-sweep", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/artifacts/profile-cards/tasks/1/verification.md", "artifactBytes": 2507, "artifactSha256": "f622bca054d281b15d56713eee137dd9b0d484cfde085ff4964e3061aea38ffe", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/co/codemod-sweep/versions/1.0.6/test.json", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ], "scoreAxes": { "Enumeration completeness": 8, "Invariant discipline": 9, "Reconciled verification": 8 }, "meanScore": 8.33, "justification": "A real pre-edit search found all seven editable sites and four exclusions. Captured files contain only the requested rename; protected files are byte-identical. Counts reconcile and the quoted test receipt is real. Verification uses file-by-file inspection rather than a recorded residual grep; the reviewer adds an incorrect tool-name nit.", "reviewBasis": "Read the complete verification and three supporting documents, all changed source/test files and the rename brief. Compared every seeded non-harness fixture to its captured final content, and inspected search, edit, test and review events. The fixture oracle already ran successfully; no extra model or test execution was performed.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/artifacts/profile-cards/tasks/1/sites.md", "bytes": 1679, "sha256": "624cbbeef94d46445af03d6a71927dab4c1e5c72abd0b6aefbdd8067a5cc2c9f", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/artifacts/profile-cards/tasks/1/change-notes.md", "bytes": 1468, "sha256": "1caf01ce08805f6d4f954e9e45871ce09b919a1c9c0951983c7627301f62b8d5", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/artifacts/profile-cards/tasks/1/review.md", "bytes": 4295, "sha256": "4c3800ce743348d76fa9dd1254e22fd03fda1e1437f0c226c346b4e2a9c20abb", "status": "present", "review": "Read as supporting deliverable; no separate undeclared axes assigned." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:12.381Z", "reviewId": "032", "cellId": "r1-saluki-cuda-developer-craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "scenarioId": "craftbook-codemod-sweep", "status": "scored", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "baselineCellId": "r1-saluki-cuda-developer-craftbook-codemod-sweep", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "ec8d64742d6144d7de225d4ae80415824d5b931ea19058570fbbccbe3f51bf7c", "evidenceSha256": "f7002983e1657052b848d3030a8e28dc25cdd60a1600a0a318127182a2eb8d6f", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/artifacts/profile-cards/tasks/1/verification.md", "bytes": 2507, "sha256": "f622bca054d281b15d56713eee137dd9b0d484cfde085ff4964e3061aea38ffe", "shownBytes": 2507, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6\\workspace\\profile-cards\\source\\rename-brief.md", "bytes": 534, "sha256": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "shownBytes": 534, "truncated": false } ], "scoreAxes": { "Enumeration completeness": 6, "Invariant discipline": 6, "Reconciled verification": 7 }, "justification": "Verification lists 7 sites (definition, re-export, import, calls, log string, test) and excludes CHANGELOG with the brief's reason. Counts reconcile 7=7+0, and 4 exclusions are accounted for. Suite output is quoted (3 pass). But sites.md and change-notes.md aren't shown, there's no diff, and the residual scan is a hand-built table, not raw search output.", "limitations": "Only verification.md is visible. The pre-edit enumeration (sites.md), change notes, diffs and raw search or test output are absent. Untouched exclusions and unchanged behavior can't be independently confirmed. Case-variant and alias searches aren't evidenced.", "meanScore": 6.333333333333333, "durationMs": 17578, "costUsd": 0.026477999999999998, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4676, "cache_read_input_tokens": 0, "output_tokens": 777, "output_tokens_details": { "thinking_tokens": 400 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4676, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 777, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 4676, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4676 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 777, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 4676, "webSearchRequests": 0, "costUSD": 0.026477999999999998, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 400, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4676, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-codemod-sweep / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 28m 5s; budget used: 15.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 340691`; `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1659,\"score\":32,\"scoreMax\":32,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.14995409000281448`; `facts.toolUse.totalToolCalls: 46`; `facts.timing.timeToFirstArtifactMs: 340691`; `facts.outcome.durationMs: 1685228` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15548 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31436 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.8 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 56535 + 12807 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 255s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 13% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 340691`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1659,\"score\":32,\"scoreMax\":32,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.14995409000281448`\n- `facts.toolUse.totalToolCalls: 46`\n- `facts.timing.timeToFirstArtifactMs: 340691`\n- `facts.outcome.durationMs: 1685228`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 340691`; `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1659,\"score\":32,\"scoreMax\":32,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.14995409000281448`; `facts.toolUse.totalToolCalls: 46`; `facts.timing.timeToFirstArtifactMs: 340691`; `facts.outcome.durationMs: 1685228`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nRead the complete verification and three supporting documents, all changed source/test files and the rename brief. Compared every seeded non-harness fixture to its captured final content, and inspected search, edit, test and review events. The fixture oracle already ran successfully; no extra model or test execution was performed.\n\nPrimary artifact: 2507 UTF-8 bytes. Reviewer: codex-inline/current-session.\n\n| Axis | Score | Exact declared description |\n|---|---:|---|\n| Enumeration completeness | 8 | Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed. |\n| Invariant discipline | 9 | The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical. |\n| Reconciled verification | 8 | The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real. |\n\n**Advisory mean: 8.33/10.**\n\nA real pre-edit search found all seven editable sites and four exclusions. Captured files contain only the requested rename; protected files are byte-identical. Counts reconcile and the quoted test receipt is real. Verification uses file-by-file inspection rather than a recorded residual grep; the reviewer adds an incorrect tool-name nit.\n\nFixed composite remains **8.5/10**; outcome remains **pass**. This advisory review does not alter the deterministic score.\n\n## Positives\n\n- All 32 deterministic checks passed and the actual craftbook reached terminal completion (outcome.success; sniff.latest.score: 32/32). The pre-edit grep_files call returned 11 match lines at 19:38:28.868Z, correctly divided into seven editable sites and four exclusions before the first edit at 19:45:11.076Z (toolUse.byTool.grep_files: 1; advisory-codemod-audit.json).\n- All four changed files equal the original fixture with only fetchUserData replaced by loadUser; package.json, src/log.js, CHANGELOG.md and the input brief are byte-identical. The re-export, import/call sites and log string were all handled without changing assertions or runtime behavior (toolUse.byTool.replace_in_file: 7; sniff.latest.score; captured fixture comparison).\n- Both real run_package_script calls exited 0 with three tests passing. The verification document reproduces the second receipt’s exact timings and reconciles seven found with seven changed plus zero remaining un-excluded sites (toolUse.byTool.run_package_script: 2; recording at 19:48:07.517Z). The Reviewer explicitly labels the test verdict as based on the recorded receipt rather than claiming a separate run.\n\n## Negatives\n\n- The Reviewer incorrectly says the enumeration named grep_files when the actual tool was grep_artifact. The trace proves grep_files was used. This is a false review nit about provenance, even though the edit and counts are correct (toolUse.byTool.grep_files: 1; final review.md).\n- The verification describes an exact whole-tree re-scan, but the trace shows batched file reads after editing rather than a second search-tool receipt. Manual inspection supports the counts in this small captured tree; the prose should identify that method precisely instead of implying an independently recorded search command (toolUse.byTool.read_files: 4; grep_files: 1).\n- The first enumeration write was refused at 19:41:52.445Z for numerical facts including the directly returned 11 matches and source line numbers; adding citations allowed a later write. A derived count of seven genuinely needs a derivation, but treating literal returned line numbers as wholly absent evidence is noisy (toolUse.byTool.write_artifact: 5 includes the refusal; advisory-tool-evidence.json).\n- Eight created roles and 46 tool calls took 28m05s for seven mechanical replacements across four files. Most of the later work was verification and review rather than further editing (team.totalGezelsCreated; toolUse.totalToolCalls; outcome.durationMs). The quality is strong, but orchestration cost remains material on this hardware.\n\n## Tactical experiment — not applied\n\nFor a future version of ../gilde/data/gezel-templates/re/reviewer/versions/1.4.1/about.md, add: “Distinguish direct tool execution from manual file inspection and inherited receipts. Quote a tool name only from its actual receipt; do not infer another worker’s tool availability from your current tool list.” Compare false provenance findings on codemods and unrelated research/citation workflows. Keep this as general reviewer guidance, with no scenario-specific identifiers.\n\n## Strategic experiment — not applied\n\nIn packages/service/src/chat/tool-evidence-replay.ts, preserve small immutable receipt records (tool name, input, time, exit code, affected paths and output hash) across role handoffs so the reviewer can inspect actual search/test execution. In packages/service/src/chat/evidence-ledger.ts, represent source line numbers and explicit search match counts as typed evidence; keep derived arithmetic labeled separately. Measure rejected supported statements and false reviewer corrections on code, data and document tasks before adopting either experiment.\n\n## Performance and fit\n\nElapsed 28.09 minutes; 12807 recorded generated tokens; mean decode 8.8 tokens/sec, prefill 392.26 tokens/sec. Peak VRAM 10638/12227 MiB, process RSS 15548 MiB, whole-system RAM 31436/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). One native 0.1.57 CUDA launch, 65536-token context, f16 KV, one slot, 4096 reasoning budget. The planner places dense FFN from 39/64 layers on GPU and 25 on CPU. This demonstrates mixed residency, not a fully GPU-resident 27B model. Shared GPU memory and paging are not separately measured. No explicit CUDA OOM, context overflow, endpoint failure or unexpected native exit was found in this trial.\n\n## Generalization and next experiment\n\nUnlike large-pr-review’s self-authored 120/120 ledger, this sweep has a pre-edit search receipt, exact mechanical output and real test executions. It also avoids the competing writers seen in deep-security-review. The recurring grounding refusal resembles code-review and A/B-readout, suggesting a broad evidence-transport issue worth testing rather than adding a special-case prompt or gate.\n\nThe concrete future experiment is to carry source/execution receipts across role handoffs, then measure evidence-backed completion claims and false corrections across code review, codemods and unrelated research tasks. No tuning, prompts, gates, weights or source code were changed in this frozen campaign.\n\n## Last three same-scenario trials\n\n- [craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/facts.json): fail, fixed 5.3, 1007.8 seconds, recorded decode 119.22 tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on workspace output; missing signals stayed unchanged: tests/profile.test.mjs did not pass when run with node: exit=1\nCommand failed: Files\\nodejs\\node.exe tests/profile.test.mjs\n/tests/profile.test.mjs:3\r\nimport { fetchUserData } from '../src/index.js';\r\n ^^^^^^^^^^^^^\r\nSyntaxError: The requested module '../src/index.js' does not provide an export named 'fetchUserData'\r\n at #asyncInstantiate (node:internal/modules/esm/module_job:455:21)\r\n at async ModuleJob.run (node:internal/modules/esm/module_job:553:5)\r\n at async node:internal/modules/esm/loader:647:26\r\n at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)\r\n\r\nNode.js v24.21.0, tests/verify-sweep.mjs did not pass when run with node: exit=1\nCommand failed: Files\\nodejs\\node.exe tests/verify-sweep.mjs\nnode:internal/modules/run_main:107\r\n triggerUncaughtException(\r\n ^\r\n\r\nAssertionError [ERR_ASSERTION]: the old name survives outside the exclusions in: tests/profile.test.mjs\r\n+ actual - expected\r\n\r\n+ [\r\n+ 'tests/profile.test.mjs'\r\n+ ]\r\n- []\r\n\r\n at /tests/verify-sweep.mjs:32:8\r\n at ModuleJob.run (node:internal/modules/esm/module_job:561:25)\r\n at async node:internal/modules/esm/loader:647:26\r\n at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) {\r\n generatedMessage: false,\r\n code: 'ERR_ASSERTION',\r\n actual: [ 'tests/profile.test.mjs' ],\r\n expected: [],\r\n operator: 'deepStrictEqual',\r\n diff: 'simple'\r\n}\r\n\r\nNode.js v24.21.0, task sourced from craftbook codemod-sweep has not reached a terminal step, tests/profile.test.mjs contains forbidden content: the old name is gone from the test call sites (matched \"fetchUserData\"). Last failure: tests/profile.test.mjs did not pass when run with node: exit=1 Command failed: Files\\nodejs\\node.exe tests/profile.test.mjs /tests/profile.test.mjs:3 import { fetchUserData } from '../src/index.js'; ^^^^^^^^^^^^^ SyntaxError: The requested module '../src/index.js' does not provide an export named 'fetchUserData' at #asyncInstantiate (node:internal/modules/esm/module_job:455:21) at async ModuleJob.run (node:internal/modules/esm/module_job:553:5) at async node:internal/modules/esm/loader:647:26 at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) Node.js v24.21.0\n- [craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/facts.json): fail, fixed 4.6, 479.2 seconds, recorded decode 109.44 tokens/sec. craftbook-codemod-sweep stale no-write repair loop: tasks/1/verification.md had unchanged content (882 bytes) for 77s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 27/32: tasks/1/verification.md is missing required content: verification sections in order — nothing in its 882 bytes matches /^##\\s+Residual scan[\\s\\S]*^##\\s+Counts reconciled[\\s\\S]*^##\\s+Suite[\\s\\S]*^##\\s+Result/. Add that content.\n- [craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-34-32-173Z-sgio/facts.json): fail, fixed 5.4, 984.7 seconds, recorded decode 121.9 tokens/sec. repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 28 while the failing detail kept changing; latest checked file workspace output. Last failure: tests/profile.test.mjs did not pass when run with node: exit=1 Command failed: Files\\nodejs\\node.exe tests/profile.test.mjs /tests/profile.test.mjs:3 import { loadUser } from '../src/index.js'; ^^^^^^^^ SyntaxError: Identifier 'loadUser' has already been declared at compileSourceTextModule (node:internal/modules/esm/utils:319:16) at ModuleLoader.moduleStrategy (node:internal/modules/esm/translators:89:18) at #translate (node:internal/modules/esm/loader:438:20) at afterLoad (node:internal/modules/esm/loader:506:29) at ModuleLoader.loadAndTranslate (node:internal/modules/esm/loader:511:12) at #getOrCreateModuleJobAfterResolve (node:internal/modules/esm/loader:564:36) at aft\n\nEarlier outcomes: 0/3 passed. Different model/native/source identities make these historical context, not a controlled native-version comparison.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json and historical-comparison.json beside this report; the specialized advisory audit preserves the coverage or fixture comparison.\n" }, { "cellId": "r1-saluki-cuda-developer-craftbook-refactor-module", "baselineCellId": "r1-saluki-cuda-developer-craftbook-refactor-module", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T20:08:16.627Z", "finishedAt": "2026-11-04T21:08:04.109Z", "durationMs": 3587482, "success": true, "reason": "craftbook-refactor-module passed 30 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 377743 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 30, "scoreMax": 30, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2394146015009886 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 50 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 377743 }, { "source": "facts", "path": "outcome.durationMs", "value": 3587482 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 599, "peakRssMb": 15075, "peakCpuPercent": 1321.1, "firstAtMs": 1, "lastAtMs": 3575397 }, "gpu": { "available": true, "sampleCount": 599, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 599, "peakUsedMb": 31427, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 77085, "totalOutputTokens": 28106, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 83806, "todayTokensCached": 76918, "todayTokensOut": 7211, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 83806, "totalTokensCached": 76918, "totalTokensOut": 7211, "totalCost": 0, "medianOutputTokensPerSec": 8.6, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.6, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8901, "cachedInputTokens": 8435, "outputTokens": 87, "cost": 0, "durationMs": 113488, "outputTokensPerSec": 8.73630381207601, "at": "2026-11-04T21:07:57.411Z" }, "lastUpdated": "2026-11-04T21:07:57.411Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T21:07:59.824Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3577001, "meanTokensPerSec": 8.6, "genTokensPerSec": 8.6, "genTokens": 28106, "promptTokensPerSec": 379.23, "peakRssMb": 15075, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 83806, "output": 7211 } } } }, "craftbook": { "craftbookId": "refactor-module", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 7, "completed": 6, "gateApprovals": 6, "gateRejections": 2, "redrives": 0, "perStep": [ { "ref": "crewlog/1", "stepId": "baseline", "activatedAt": "2026-11-04T20:08:24.156Z", "completedAt": "2026-11-04T20:17:35.644Z", "ms": 551488, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "bd77b30c-07c3-4346-92f4-fe1cb169076d" ] }, { "ref": "crewlog/1", "stepId": "plan", "activatedAt": "2026-11-04T20:21:45.998Z", "completedAt": "2026-11-04T20:24:48.058Z", "ms": 182060, "gateRejections": 1, "toolCalls": 4, "sessionIds": [ "f447dd8b-dc03-4369-b84e-a06feadbd661" ] }, { "ref": "crewlog/1", "stepId": "change", "activatedAt": "2026-11-04T20:24:48.060Z", "completedAt": "2026-11-04T20:36:11.337Z", "ms": 683277, "gateRejections": 0, "toolCalls": 16, "sessionIds": [ "31dd2388-ad33-4f5a-ab83-4cd633c12ccb" ] }, { "ref": "crewlog/1", "stepId": "verify", "activatedAt": "2026-11-04T20:36:11.338Z", "completedAt": "2026-11-04T20:43:54.942Z", "ms": 463604, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "31dd2388-ad33-4f5a-ab83-4cd633c12ccb" ] }, { "ref": "crewlog/1", "stepId": "evaluate", "activatedAt": "2026-11-04T21:02:53.668Z", "completedAt": "2026-11-04T21:05:57.185Z", "ms": 183517, "gateRejections": 1, "toolCalls": 17, "sessionIds": [ "6d7290fd-b596-4f91-af65-ca3a7dbfd657" ] }, { "ref": "crewlog/1", "stepId": "finish", "activatedAt": "2026-11-04T21:05:57.187Z", "completedAt": "2026-11-04T21:07:57.369Z", "ms": 120182, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "1c5db1b9-6cde-4b3b-a423-9a85eae4244f" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T20:08:22.360Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T20:08:22.265Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 323560.5 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "ahou": 1, "dimitrije": 2, "inna": 1, "nalani": 1 }, "perTask": { "crewlog/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.166229248046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.166229248046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-refactor-module passed 30 deterministic craftbook checks", "durationMs": 3587482, "timeoutMs": 14984391, "budgetUsedFraction": 0.2394146015009886 }, "timing": { "startedAt": "2026-11-04T20:08:16.627Z", "finishedAt": "2026-11-04T21:08:04.109Z", "timeToFirstArtifactMs": 377743, "timeToLastArtifactWriteMs": 3460588, "timeToFirstTokenMs": 35888, "firstTurnTtftMs": 22457, "timeToFirstToolCallMs": 49067 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Developer", "Software Architect", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 50, "byTool": { "read_files": 4, "run_package_script": 5, "write_file": 4, "write_artifact": 7, "read_artifact": 9, "grep_artifact": 1, "read_file": 7, "replace_lines": 1, "replace_in_file": 2, "list_dir": 1, "validate": 4, "read_task_notes": 1, "advance_task_step": 3, "write_task_note": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 17 }, "sniff": { "progression": [ { "atMs": 8003, "filePath": "craftbook-refactor-module", "bytes": 0, "score": 5, "scoreMax": 30, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/format-name.js, src/report.js, tests/format.test.mjs, tests/report.test.mjs, source/refactor-brief.md" }, { "atMs": 54393, "filePath": "craftbook-refactor-module", "bytes": 0, "score": 6, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 559227, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 10, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 814997, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 14, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 994112, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 15, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1393747, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 15, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1537168, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 16, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1585460, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 18, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1675480, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 22, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 2143275, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 26, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 3267084, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 28, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 3279063, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 28, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 3571224, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 29, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 3582769, "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 30, "scoreMax": 30, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-refactor-module", "bytes": 3355, "score": 30, "scoreMax": 30, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T20:08:16.632Z] [trial] id=craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5 scenario=craftbook-refactor-module model=underdog-saluki-27b-iq2-mix", "[2026-11-04T20:08:16.632Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "[2026-11-04T20:08:16.632Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-refactor-module-iG1CVG", "[2026-11-04T20:08:16.632Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T20:08:16.632Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T20:08:16.635Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T20:08:16.640Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T20:08:16.645Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-refactor-module-iG1CVG", "[2026-11-04T20:08:16.645Z] [trial] throughput-scaled ceiling: 120m → 250m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T20:08:22.356Z] [trial] daemon spawned pid=44720 port=https://127.0.0.1:57377", "[2026-11-04T20:08:23.713Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-04T20:08:23.728Z] [trial] running scenario setup hook", "[2026-11-04T20:08:23.728Z] [trial] meester=malika", "[2026-11-04T20:08:24.327Z] [trial] scenario setup complete", "[2026-11-04T20:08:24.329Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T20:08:24.391Z] [poll] initial digests hard=6f8914a98e563b1c soft=06f67dfdeed8225b", "[2026-11-04T20:08:24.730Z] [poll] hard-progress: 6f8914a98e563b1c → fa4f4ef694235a4c after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-04T20:08:30.184Z] [poll] hard-progress: fa4f4ef694235a4c → 9711380c34429a7a after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-04T20:09:11.086Z] [poll] hard-progress: 9711380c34429a7a → 125d11994cc1024d after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=46,stream=161) sniff=6/0B)", "[2026-11-04T20:10:21.955Z] [poll] hard-progress: 125d11994cc1024d → b66a1b24c4648342 after 71s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=235,stream=718) sniff=6/0B)", "[2026-11-04T20:14:36.461Z] [poll] hard-progress: b66a1b24c4648342 → 383d2b90491d77f6 after 255s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=952,stream=3303) sniff=6/0B)", "[2026-11-04T20:15:00.461Z] [poll] hard-progress: 383d2b90491d77f6 → 64aeee3385c7c8ab after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1007,stream=3464) sniff=6/0B)", "[2026-11-04T20:17:35.909Z] [poll] hard-progress: 64aeee3385c7c8ab → 5921c72e18400186 after 155s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=1439,stream=5645) sniff=10/3355B)", "[2026-11-04T20:17:41.729Z] [poll] hard-progress: 5921c72e18400186 → 73f6f12e6fe56a45 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=1439,stream=5645) sniff=10/3355B)", "[2026-11-04T20:18:11.263Z] [poll] hard-progress: 73f6f12e6fe56a45 → a81373e7b07b3ea9 after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=1462,stream=5690) sniff=10/3355B)" ] }, "metrics": { "process": { "sampleCount": 599, "peakRssMb": 15075, "peakCpuPercent": 1321.1, "firstAtMs": 1, "lastAtMs": 3575397 }, "gpu": { "available": true, "sampleCount": 599, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 599, "peakUsedMb": 31427, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 77085, "totalOutputTokens": 28106, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 83806, "todayTokensCached": 76918, "todayTokensOut": 7211, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 83806, "totalTokensCached": 76918, "totalTokensOut": 7211, "totalCost": 0, "medianOutputTokensPerSec": 8.6, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.6, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8901, "cachedInputTokens": 8435, "outputTokens": 87, "cost": 0, "durationMs": 113488, "outputTokensPerSec": 8.73630381207601, "at": "2026-11-04T21:07:57.411Z" }, "lastUpdated": "2026-11-04T21:07:57.411Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T21:07:59.824Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3577001, "meanTokensPerSec": 8.6, "genTokensPerSec": 8.6, "genTokens": 28106, "promptTokensPerSec": 379.23, "peakRssMb": 15075, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 83806, "output": 7211 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T20:08:16.627Z", "finishedAt": "2026-11-04T21:08:04.109Z", "timeToFirstArtifactMs": 377743, "timeToLastArtifactWriteMs": 3460588, "timeToFirstTokenMs": 35888, "firstTurnTtftMs": 22457, "timeToFirstToolCallMs": 49067 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 7, "completed": 6, "gateApprovals": 6, "gateRejections": 2, "redrives": 0, "perStep": [ { "ref": "crewlog/1", "stepId": "baseline", "activatedAt": "2026-11-04T20:08:24.156Z", "completedAt": "2026-11-04T20:17:35.644Z", "ms": 551488, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "bd77b30c-07c3-4346-92f4-fe1cb169076d" ] }, { "ref": "crewlog/1", "stepId": "plan", "activatedAt": "2026-11-04T20:21:45.998Z", "completedAt": "2026-11-04T20:24:48.058Z", "ms": 182060, "gateRejections": 1, "toolCalls": 4, "sessionIds": [ "f447dd8b-dc03-4369-b84e-a06feadbd661" ] }, { "ref": "crewlog/1", "stepId": "change", "activatedAt": "2026-11-04T20:24:48.060Z", "completedAt": "2026-11-04T20:36:11.337Z", "ms": 683277, "gateRejections": 0, "toolCalls": 16, "sessionIds": [ "31dd2388-ad33-4f5a-ab83-4cd633c12ccb" ] }, { "ref": "crewlog/1", "stepId": "verify", "activatedAt": "2026-11-04T20:36:11.338Z", "completedAt": "2026-11-04T20:43:54.942Z", "ms": 463604, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "31dd2388-ad33-4f5a-ab83-4cd633c12ccb" ] }, { "ref": "crewlog/1", "stepId": "evaluate", "activatedAt": "2026-11-04T21:02:53.668Z", "completedAt": "2026-11-04T21:05:57.185Z", "ms": 183517, "gateRejections": 1, "toolCalls": 17, "sessionIds": [ "6d7290fd-b596-4f91-af65-ca3a7dbfd657" ] }, { "ref": "crewlog/1", "stepId": "finish", "activatedAt": "2026-11-04T21:05:57.187Z", "completedAt": "2026-11-04T21:07:57.369Z", "ms": 120182, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "1c5db1b9-6cde-4b3b-a423-9a85eae4244f" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T20:08:22.360Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T20:08:22.265Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 323560.5 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "ahou": 1, "dimitrije": 2, "inna": 1, "nalani": 1 }, "perTask": { "crewlog/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.166229248046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.166229248046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-craftbook-refactor-module", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "scenarioId": "craftbook-refactor-module", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/artifacts/crewlog/tasks/1/change-notes.md", "artifactBytes": 1274, "artifactSha256": "69717175f34a2f3ac3856869d31ac8508729ef275ac80308a05184ec8a912bc3", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/re/refactor-module/versions/2.0.7/test.json", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ], "scoreAxes": { "Guardrail-first discipline": 8, "Reversible-stage execution": 6, "Genuine deduplication": 9 }, "meanScore": 7.67, "justification": "Real baseline runs and ten new characterization tests precede restructuring. The shared implementation preserves behavior and exports. The notes invent a separate stage-1 green run and list unchanged files as touched; the reviewer accepts those claims. Final tests and deduplication are genuine, but execution reporting is less reliable.", "reviewBasis": "Read all five final documents, compared final source/test content to the seeded fixtures and earlier checkpoint, and inspected the full recorded test/edit chronology. No new test execution or model retry was performed.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/artifacts/crewlog/tasks/1/baseline.md", "bytes": 3388, "sha256": "79b996b4588ed266a684fa6403a72a24ce120a929e33a46012807380a2348639", "status": "present", "review": "Read supporting document; no extra undeclared axes." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/artifacts/crewlog/tasks/1/plan.md", "bytes": 5234, "sha256": "534d746e7b7b6350f82fc0e26298410bfbdd092922a0b4173e30a13210774b3b", "status": "present", "review": "Read supporting document; no extra undeclared axes." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/artifacts/crewlog/tasks/1/verification.md", "bytes": 4316, "sha256": "ebe9aeeb74458ae469c4cd3732af0729e49a08c32126dd8b5447d7e69d91a4b2", "status": "present", "review": "Read supporting document; no extra undeclared axes." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/artifacts/crewlog/tasks/1/review.md", "bytes": 4706, "sha256": "4164886f229dc2a638ca5d8d5f59795d66fdfd3310118cd30cb94726ce461fa9", "status": "present", "review": "Read supporting document; no extra undeclared axes." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:17.749Z", "reviewId": "033", "cellId": "r1-saluki-cuda-developer-craftbook-refactor-module", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "scenarioId": "craftbook-refactor-module", "status": "scored", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "baselineCellId": "r1-saluki-cuda-developer-craftbook-refactor-module", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "3b19f8c9f065e18328b3669f3108c4c6d2146318bd050c3495e4828d6a4449bb", "evidenceSha256": "0772f10b3f7a9c90fde50edc1b55735d60dce96c9c54d7fe7d44bdf71d6c9ed9", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/artifacts/crewlog/tasks/1/change-notes.md", "bytes": 1274, "sha256": "69717175f34a2f3ac3856869d31ac8508729ef275ac80308a05184ec8a912bc3", "shownBytes": 1274, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5\\workspace\\crewlog\\source\\refactor-brief.md", "bytes": 720, "sha256": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "shownBytes": 720, "truncated": false } ], "scoreAxes": { "Guardrail-first discipline": 4, "Reversible-stage execution": 7, "Genuine deduplication": 7 }, "justification": "Three small stages (additive module, rewire format-name, rewire report) each followed by a green 12/12 run, with 'none' for deviations. Both sites delegate to normalize-name.js and reportSummary is untouched. But no baseline record is shown, with no pre-change green run, no duplication count and no method. package.json is touched without explanation.", "limitations": "Only change-notes were visible. Baseline output, the characterization test contents, the diffs and the package.json change were not shown, and no test output could be checked. Edits during a baseline step may also be out of scope.", "meanScore": 6, "durationMs": 17996, "costUsd": 0.02454, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4304, "cache_read_input_tokens": 0, "output_tokens": 732, "output_tokens_details": { "thinking_tokens": 393 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4304, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 732, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 4304, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4304 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 732, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 4304, "webSearchRequests": 0, "costUSD": 0.02454, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 393, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4304, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-refactor-module / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 59m 47s; budget used: 23.9%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 377743`; `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":3355,\"score\":30,\"scoreMax\":30,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2394146015009886`; `facts.toolUse.totalToolCalls: 50`; `facts.timing.timeToFirstArtifactMs: 377743`; `facts.outcome.durationMs: 3587482` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15075 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31427 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.6 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 77085 + 28106 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 8 → 6 | `continuity.steps` |\n| Gate rejections / redrives | 2 / 0 | `continuity.steps` |\n| Median step wall-clock | 324s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.63 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 17% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 377743`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":3355,\"score\":30,\"scoreMax\":30,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.2394146015009886`\n- `facts.toolUse.totalToolCalls: 50`\n- `facts.timing.timeToFirstArtifactMs: 377743`\n- `facts.outcome.durationMs: 3587482`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 377743`; `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":3355,\"score\":30,\"scoreMax\":30,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2394146015009886`; `facts.toolUse.totalToolCalls: 50`; `facts.timing.timeToFirstArtifactMs: 377743`; `facts.outcome.durationMs: 3587482`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nRead all five final documents, compared final source/test content to the seeded fixtures and earlier checkpoint, and inspected the full recorded test/edit chronology. No new test execution or model retry was performed.\n\nFinal primary artifact: 1274 UTF-8 bytes, unchanged from the provisional checkpoint. Exact declared axes:\n\n| Axis | Score | Description |\n|---|---:|---|\n| Guardrail-first discipline | 8 | The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method. |\n| Reversible-stage execution | 6 | The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over. |\n| Genuine deduplication | 9 | One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor. |\n\n**Advisory mean: 7.67/10, codex-inline/current-session. Fixed composite: 8.5/10, pass.**\n\nReal baseline runs and ten new characterization tests precede restructuring. The shared implementation preserves behavior and exports. The notes invent a separate stage-1 green run and list unchanged files as touched; the reviewer accepts those claims. Final tests and deduplication are genuine, but execution reporting is less reliable.\n\n## Positives\n\n- The actual craftbook reached terminal completion and all 30 deterministic checks passed (outcome.success; sniff.latest.score: 30/30; craftbook.reachedTerminal). The source contains one normalization implementation, with both original callers delegating and their public exports preserved.\n- An initial two-test run passed before any edit, then ten characterization tests were added and all twelve passed before restructuring. The tests pin empty/whitespace-only names, case normalization, inner spaces, single letters and report behavior (toolUse.byTool.run_package_script: 5; advisory-execution-audit.json). The two original tests remain byte-identical to the fixtures.\n- The shared function body is copied from the original implementation, without an observable logic change; package.json and the task brief are unchanged. All five recorded test invocations passed, including a final twelve-test run at 20:40:12.931Z (sniff.latest.score; captured sources; test receipts).\n\n## Negatives\n\n- The final change notes claim a green test run after each of three stages, but the recording shows the new module at 20:31:30.119Z, caller rewiring at 20:33:52.571Z, then the next test at 20:34:09.811Z. There was no intervening stage-1-only test. The reviewer accepts the author’s claim as a receipt, and Deviations from plan remains “none” (toolUse.byTool.run_package_script: 5; recording has 0 gaps; advisory-execution-audit.json).\n- Files touched lists package.json, the brief and both original test files even though they are byte-identical. The plan misallocates the twelve tests as 2+1+9, while the real split is 1+1+10; the baseline has the correct characterization count. These are documentation defects rather than a source or test failure (artifacts.otherFileCount; final documents and fixture hashes).\n- Two write_file attempts and one whole-file replace_lines attempt were refused because the legitimate wrapper shrank from 240 to 119 bytes, despite a preceding source read. An exact replace_in_file succeeded. This is overbroad truncation protection that cost three failed calls (toolUse.byTool.write_file: 4; replace_lines: 1; recorded refusal text).\n- The plan gate initially rejected a proposed new path because it did not yet exist; the author removed backticks around it. Later the review gate rejected a no-findings table row with severity “—”; replacing that row with prose cleared it. These formatting repairs did not change source behavior (modes.steps.gateRejections: 2; captured plan/review writes and gate messages).\n- Eight created roles, 50 tool calls and 28106 generated tokens took 59m47s for a small extraction. The review arrived about 19 minutes after final verification and still missed the test-receipt discrepancy (team.totalGezelsCreated; toolUse.totalToolCalls; perf.derived.genTokens; outcome.durationMs).\n\n## Tactical experiment — not applied\n\nIn a future version of ../gilde/data/gezel-templates/re/reviewer/versions/1.4.1/about.md, add: “Grade each claimed command from its execution receipt and timestamp, not a GREEN label in the author’s notes. List files as modified only when their content changed; distinguish read-only inspection. Missing intermediate verification is a plan deviation even if the final suite passes.” Measure unsupported execution claims on refactors, database migrations and unrelated document-conversion workflows. This is general evidence guidance, with no scenario-specific strings.\n\n## Strategic experiment — not applied\n\nIn packages/core/src/local-loop/wrappers/source-write-guard.ts, revise the short-overwrite error path to direct an available replace_in_file call using an exact old-content match, instead of suggesting a whole-file replace_lines or reread/full-write that the same heuristic will refuse again. Keep placeholder/truncation checks intact; validate the change on legitimate extraction, ordinary source repair and truncated HTML writes. In packages/service/src/chat/tool-evidence-replay.ts, preserve concise immutable command/exit/timestamp receipts across role handoffs so the reviewer can verify stages independently of prose summaries.\n\n## Performance and fit\n\nElapsed 59.79 minutes; 28106 recorded generated tokens; mean decode 8.6 tokens/sec, prefill 379.23 tokens/sec. Peak VRAM 10646/12227 MiB, process RSS 15075 MiB, whole-system RAM 31427/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). One native 0.1.57 CUDA launch, 65536 context, f16 KV, one slot, 4096 reasoning budget. The planner places dense FFN from 39/64 layers on GPU and 25 on CPU. This is mixed residency, not full GPU residency; shared GPU memory and paging are not separately measured. No explicit CUDA OOM, context overflow, endpoint failure or unexpected native exit was found.\n\n## Generalization and recommended next experiment\n\nThe unrelated codemod-sweep also made a correct mechanical edit, but its Reviewer misstated the search tool name. Large-pr-review claimed 120 reviewed records without corresponding read evidence. Here the final source works while an intermediate command claim is unsupported. Carrying actual execution/read receipts into reviewer turns is a cross-task experiment in provenance, not a prompt designed to memorize this extraction. Measure false completion claims and unsupported reviewer corrections across code, research and document work while keeping their existing success gates fixed. No tuning, gates, model weights, source code or automatic retry policy was changed.\n\n## Last three same-scenario trials\n\n- [craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/facts.json): fail, fixed 5, 1590.6 seconds, 120.37 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on tasks/1/plan.md; missing signals stayed unchanged: tasks/1/plan.md is missing required content: plan sections in order — nothing in its 1829 bytes matches /^##\\s+Target[\\s\\S]*^##\\s+Stages[\\s\\S]*^##\\s+Acceptance criteria/. Add that content.. Last failure: tasks/1/plan.md is missing required content: plan sections in order — nothing in its 1829 bytes matches /^##\\s+Target[\\s\\S]*^##\\s+Stages[\\s\\S]*^##\\s+Acceptance criteria/. Add that content.\n- [craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/facts.json): fail, fixed 4.7, 804.2 seconds, 129.41 recorded tokens/sec. craftbook-refactor-module stale no-write repair loop: tasks/1/plan.md had unchanged content (745 bytes) for 150s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 18/30: tasks/1/plan.md cites 1 source(s) that do not exist: `src/normalize-name.js` — every cited path must resolve to a real file in the workspace (no fabricated citations).\n- [craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-15-49-563Z-7mct/facts.json): fail, fixed 4.8, 707.3 seconds, 128.83 recorded tokens/sec. craftbook-refactor-module stale no-write repair loop: tasks/1/plan.md had unchanged content (1960 bytes) for 127s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 19/30: tasks/1/plan.md cites 1 source(s) that do not exist: `./normalize-name.js` — every cited path must resolve to a real file in the workspace (no fabricated citations).\n\nEarlier outcomes: 0/3 passed. Model/native/source identities differ, so these are historical context, not a controlled native-version comparison. Full reasons are preserved in historical-comparison.json.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json and advisory-execution-audit.json beside this report. The earlier advisory-progress-review.json is retained as a checkpoint.\n\n## Strategic note\n\nTwo of the three historical refactor trials also stalled because their plans cited the intentionally new normalization module before it existed; the third failed the required plan-section shape. This run passed after removing backticks from the planned new path. That recurring citation distinction is a framework issue, not evidence that the proposed extraction was wrong. A future experiment in packages/core/src/checks/grounding.ts should distinguish references to existing evidence from explicitly declared future outputs in plans, while retaining strict resolution for claimed existing sources. Verify it on unrelated file-creation and document-production plans before adoption; no gate was changed here.\n" }, { "cellId": "r1-saluki-cuda-developer-craftbook-bug-fix-tdd", "baselineCellId": "r1-saluki-cuda-developer-craftbook-bug-fix-tdd", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T21:11:12.352Z", "finishedAt": "2026-11-04T21:34:14.942Z", "durationMs": 1382590, "success": true, "reason": "craftbook-bug-fix-tdd passed 28 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 85026 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07381494418004783 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 36 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 85026 }, { "source": "facts", "path": "outcome.durationMs", "value": 1382590 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 231, "peakRssMb": 14934, "peakCpuPercent": 1202.8, "firstAtMs": 1, "lastAtMs": 1368319 }, "gpu": { "available": true, "sampleCount": 231, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 231, "peakUsedMb": 31274, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 66590, "totalOutputTokens": 9498, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 66502, "todayTokensCached": 63248, "todayTokensOut": 2975, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 66502, "totalTokensCached": 63248, "totalTokensOut": 2975, "totalCost": 0, "medianOutputTokensPerSec": 8.6, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.6, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11023, "cachedInputTokens": 10492, "outputTokens": 121, "cost": 0, "durationMs": 196912, "outputTokensPerSec": 8.446909659386112, "at": "2026-11-04T21:34:04.254Z" }, "lastUpdated": "2026-11-04T21:34:04.254Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T21:34:09.008Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1370439, "meanTokensPerSec": 8.58, "genTokensPerSec": 8.58, "genTokens": 9498, "promptTokensPerSec": 388.52, "peakRssMb": 14934, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 66502, "output": 2975 } } } }, "craftbook": { "craftbookId": "bug-fix-tdd", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 6, "completed": 6, "gateApprovals": 6, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "orderly-pricing/1", "stepId": "reproduce", "activatedAt": "2026-11-04T21:11:19.820Z", "completedAt": "2026-11-04T21:13:59.970Z", "ms": 160150, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "7e1a8056-2468-4822-b442-12941a02d36f" ] }, { "ref": "orderly-pricing/1", "stepId": "locate", "activatedAt": "2026-11-04T21:13:59.972Z", "completedAt": "2026-11-04T21:18:05.436Z", "ms": 245464, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "909cbd4f-3e33-4caf-a462-304d1d6c965f" ] }, { "ref": "orderly-pricing/1", "stepId": "fix", "activatedAt": "2026-11-04T21:18:05.437Z", "completedAt": "2026-11-04T21:20:58.223Z", "ms": 172786, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "00559775-0e6f-4716-9cae-f4fd91b226a4" ] }, { "ref": "orderly-pricing/1", "stepId": "validate", "activatedAt": "2026-11-04T21:20:58.224Z", "completedAt": "2026-11-04T21:23:13.461Z", "ms": 135237, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "00559775-0e6f-4716-9cae-f4fd91b226a4" ] }, { "ref": "orderly-pricing/1", "stepId": "evaluate", "activatedAt": "2026-11-04T21:28:18.344Z", "completedAt": "2026-11-04T21:30:40.455Z", "ms": 142111, "gateRejections": 1, "toolCalls": 9, "sessionIds": [ "7cd8d95d-3c9e-4b16-bd52-eabd85ccafca" ] }, { "ref": "orderly-pricing/1", "stepId": "finish", "activatedAt": "2026-11-04T21:30:40.456Z", "completedAt": "2026-11-04T21:34:04.191Z", "ms": 203735, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "b44c5274-9bdd-407d-bbd1-ecd649d1f126" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T21:11:18.241Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T21:11:18.141Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 166468 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "aman": 1, "laxmi": 2, "sela": 1, "zoltan": 1 }, "perTask": { "orderly-pricing/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.136199951171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.136199951171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-bug-fix-tdd passed 28 deterministic craftbook checks", "durationMs": 1382590, "timeoutMs": 18730489, "budgetUsedFraction": 0.07381494418004783 }, "timing": { "startedAt": "2026-11-04T21:11:12.352Z", "finishedAt": "2026-11-04T21:34:14.942Z", "timeToFirstArtifactMs": 85026, "timeToLastArtifactWriteMs": 1168132, "timeToFirstTokenMs": 35731, "firstTurnTtftMs": 21714, "timeToFirstToolCallMs": 47402 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Developer", "Root-Cause Analyst", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 36, "byTool": { "read_files": 3, "append_to_file": 1, "run_package_script": 2, "write_artifact": 7, "read_artifact": 12, "read_file": 6, "replace_in_file": 1, "advance_task_step": 3, "write_task_note": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 13 }, "sniff": { "progression": [ { "atMs": 8017, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/pricing.js, tests/pricing.test.mjs, source/bug-report.md" }, { "atMs": 48853, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 6, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 90074, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 95765, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 101767, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 107623, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 113516, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 119400, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 125275, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 131173, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 137084, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 142984, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 148914, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 154829, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 160726, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 166637, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 172391, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 178990, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 184861, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 191075, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 197057, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 202933, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 208872, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 214786, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 220700, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 226715, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 232648, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 238548, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 244445, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 250342, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 256288, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 262216, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 268130, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 274021, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 279918, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 285819, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 291714, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 297594, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 303285, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 309286, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 315178, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 321259, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 327074, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 332978, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 338903, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 344796, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 350706, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 356636, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 362521, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 368433, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 374387, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 380399, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 386348, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 392370, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 398325, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 404310, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 410270, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 415986, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 422036, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 427883, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 433829, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 439847, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 445828, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 451761, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 457664, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 463554, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 469452, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 475333, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 481246, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 487181, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 493528, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 16, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 588931, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 20, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 725303, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 23, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1017553, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 25, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1029575, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 26, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1358706, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 27, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1376276, "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-bug-fix-tdd", "bytes": 1603, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T21:11:12.355Z] [trial] id=craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h scenario=craftbook-bug-fix-tdd model=underdog-saluki-27b-iq2-mix", "[2026-11-04T21:11:12.355Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-bug-fix-tdd-Hmth5D", "[2026-11-04T21:11:12.356Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T21:11:12.355Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "[2026-11-04T21:11:12.355Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T21:11:12.358Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T21:11:12.362Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T21:11:12.368Z] [trial] throughput-scaled ceiling: 150m → 312m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T21:11:12.367Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-bug-fix-tdd-Hmth5D", "[2026-11-04T21:11:18.212Z] [trial] daemon spawned pid=21756 port=https://127.0.0.1:57559", "[2026-11-04T21:11:19.422Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-04T21:11:19.440Z] [trial] meester=imara", "[2026-11-04T21:11:19.440Z] [trial] running scenario setup hook", "[2026-11-04T21:11:20.011Z] [trial] scenario setup complete", "[2026-11-04T21:11:20.012Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T21:11:20.070Z] [poll] initial digests hard=4cb8f11b33f16f8e soft=e43a3b3cb706b085", "[2026-11-04T21:11:20.465Z] [poll] hard-progress: 4cb8f11b33f16f8e → 8dbe0ec01fbb400b after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-04T21:11:25.824Z] [poll] hard-progress: 8dbe0ec01fbb400b → 5562c5d85e0284fb after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-04T21:12:01.264Z] [poll] hard-progress: 5562c5d85e0284fb → c7717d140135b724 after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=34,stream=106) sniff=6/0B)", "[2026-11-04T21:12:42.497Z] [poll] hard-progress: c7717d140135b724 → f03bcb502dacbba2 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=137,stream=441) sniff=5/0B)", "[2026-11-04T21:12:48.178Z] [poll] hard-progress: f03bcb502dacbba2 → 876df11bb42f5646 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=143,stream=456) sniff=5/0B)", "[2026-11-04T21:14:04.795Z] [poll] hard-progress: 876df11bb42f5646 → 2870acbf234ed4c0 after 77s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=4,slot=332,stream=1498) sniff=9/1603B)", "[2026-11-04T21:14:41.294Z] [poll] hard-progress: 2870acbf234ed4c0 → f39b354be459a17f after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=365,stream=1569) sniff=9/1603B)", "[2026-11-04T21:14:47.193Z] [poll] hard-progress: f39b354be459a17f → f0e0b85408bd96b2 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=372,stream=1592) sniff=9/1603B)", "[2026-11-04T21:14:59.120Z] [poll] hard-progress: f0e0b85408bd96b2 → e5bfb86a76c33685 after 12s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=392,stream=1654) sniff=9/1603B)" ] }, "metrics": { "process": { "sampleCount": 231, "peakRssMb": 14934, "peakCpuPercent": 1202.8, "firstAtMs": 1, "lastAtMs": 1368319 }, "gpu": { "available": true, "sampleCount": 231, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 231, "peakUsedMb": 31274, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 66590, "totalOutputTokens": 9498, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 66502, "todayTokensCached": 63248, "todayTokensOut": 2975, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 66502, "totalTokensCached": 63248, "totalTokensOut": 2975, "totalCost": 0, "medianOutputTokensPerSec": 8.6, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.6, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11023, "cachedInputTokens": 10492, "outputTokens": 121, "cost": 0, "durationMs": 196912, "outputTokensPerSec": 8.446909659386112, "at": "2026-11-04T21:34:04.254Z" }, "lastUpdated": "2026-11-04T21:34:04.254Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T21:34:09.008Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1370439, "meanTokensPerSec": 8.58, "genTokensPerSec": 8.58, "genTokens": 9498, "promptTokensPerSec": 388.52, "peakRssMb": 14934, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 66502, "output": 2975 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T21:11:12.352Z", "finishedAt": "2026-11-04T21:34:14.942Z", "timeToFirstArtifactMs": 85026, "timeToLastArtifactWriteMs": 1168132, "timeToFirstTokenMs": 35731, "firstTurnTtftMs": 21714, "timeToFirstToolCallMs": 47402 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 6, "completed": 6, "gateApprovals": 6, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "orderly-pricing/1", "stepId": "reproduce", "activatedAt": "2026-11-04T21:11:19.820Z", "completedAt": "2026-11-04T21:13:59.970Z", "ms": 160150, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "7e1a8056-2468-4822-b442-12941a02d36f" ] }, { "ref": "orderly-pricing/1", "stepId": "locate", "activatedAt": "2026-11-04T21:13:59.972Z", "completedAt": "2026-11-04T21:18:05.436Z", "ms": 245464, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "909cbd4f-3e33-4caf-a462-304d1d6c965f" ] }, { "ref": "orderly-pricing/1", "stepId": "fix", "activatedAt": "2026-11-04T21:18:05.437Z", "completedAt": "2026-11-04T21:20:58.223Z", "ms": 172786, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "00559775-0e6f-4716-9cae-f4fd91b226a4" ] }, { "ref": "orderly-pricing/1", "stepId": "validate", "activatedAt": "2026-11-04T21:20:58.224Z", "completedAt": "2026-11-04T21:23:13.461Z", "ms": 135237, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "00559775-0e6f-4716-9cae-f4fd91b226a4" ] }, { "ref": "orderly-pricing/1", "stepId": "evaluate", "activatedAt": "2026-11-04T21:28:18.344Z", "completedAt": "2026-11-04T21:30:40.455Z", "ms": 142111, "gateRejections": 1, "toolCalls": 9, "sessionIds": [ "7cd8d95d-3c9e-4b16-bd52-eabd85ccafca" ] }, { "ref": "orderly-pricing/1", "stepId": "finish", "activatedAt": "2026-11-04T21:30:40.456Z", "completedAt": "2026-11-04T21:34:04.191Z", "ms": 203735, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "b44c5274-9bdd-407d-bbd1-ecd649d1f126" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T21:11:18.241Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T21:11:18.141Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 166468 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "aman": 1, "laxmi": 2, "sela": 1, "zoltan": 1 }, "perTask": { "orderly-pricing/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.136199951171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.136199951171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-developer-craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "scenarioId": "craftbook-bug-fix-tdd", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/artifacts/orderly-pricing/tasks/1/fix-notes.md", "artifactBytes": 2246, "artifactSha256": "9d2b819755b277242067b7f75df80e9af8dcb8ad4716b2b797c2d72a858a353e", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/bu/bug-fix-tdd/versions/2.0.7/test.json", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ], "scoreAxes": { "Repro-first discipline": 9, "Fix minimality": 9, "Honest verification": 7 }, "meanScore": 8.33, "justification": "A real red run precedes the one-character boundary fix; a fresh run then passes all four tests, preserving the original assertions. Notes omit the modified test from Files touched and incorrectly discuss no per-line rounding before an unimplemented summation. The reviewer accepts those claims. Core repair and red-to-green proof are sound.", "reviewBasis": "Read all five final deliverables and source/test fixtures; compared final hashes against the provisional review and verified exact red/green receipts and edit order. The final review only removes its invalid empty-findings row. No extra test execution or model retry.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/artifacts/orderly-pricing/tasks/1/repro.md", "bytes": 1625, "sha256": "f90eb6c67f3cb3f936f2f910648c80c996ddb75cc80900acb7ef84728e7ac012", "status": "present", "review": "Read supporting document; no additional undeclared axes." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/artifacts/orderly-pricing/tasks/1/diagnosis.md", "bytes": 2368, "sha256": "5edbd12fc42b09580cde1344c2f1079fdcbb5d630c0e4989a05a26c303e68e68", "status": "present", "review": "Read supporting document; no additional undeclared axes." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/artifacts/orderly-pricing/tasks/1/validation.md", "bytes": 1785, "sha256": "2cd11292f585d7d470b90eaeb2b49ab14351cd3d7f1f5322ceac5299c2fff3b6", "status": "present", "review": "Read supporting document; no additional undeclared axes." }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/artifacts/orderly-pricing/tasks/1/review.md", "bytes": 3607, "sha256": "ae420ef70b38eef89ea5431e6f847f679de8dce6469f4b195ac685ebe5ce4504", "status": "present", "review": "Read supporting document; no additional undeclared axes." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:16.473Z", "reviewId": "034", "cellId": "r1-saluki-cuda-developer-craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "scenarioId": "craftbook-bug-fix-tdd", "status": "scored", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "baselineCellId": "r1-saluki-cuda-developer-craftbook-bug-fix-tdd", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "3350a90b60913866f5fe5d12291b993e0b1b68e5fa6a09f261b66bd0c8ece363", "evidenceSha256": "0578b463c2a64859cb85c5d67c2665cf913a9cdc81ef32484e9a44125cebb0a2", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/artifacts/orderly-pricing/tasks/1/fix-notes.md", "bytes": 2246, "sha256": "9d2b819755b277242067b7f75df80e9af8dcb8ad4716b2b797c2d72a858a353e", "shownBytes": 2246, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h\\workspace\\orderly-pricing\\source\\bug-report.md", "bytes": 313, "sha256": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "shownBytes": 313, "truncated": false } ], "scoreAxes": { "Repro-first discipline": 5, "Fix minimality": 8, "Honest verification": 4 }, "justification": "Notes describe a one-character `>` to `>=` fix at the real site in lineTotal, with no special-casing, and a test in tests/pricing.test.mjs. But the red run is only asserted (\"80 !== 68\"), not quoted from a run. \"Expected: all four tests pass\" is a prediction, not a recorded green. Reproduce-step notes also cover the fix.", "limitations": "Only fix-notes.md and the bug report were visible. I could not see the test file, the diff, or any run receipts, so the claimed red and green runs and the minimal diff are unverified. I did not run anything.", "meanScore": 5.666666666666667, "durationMs": 16233, "costUsd": 0.023218, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4301, "cache_read_input_tokens": 0, "output_tokens": 601, "output_tokens_details": { "thinking_tokens": 260 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4301, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 601, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 4301, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4301 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 601, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 4301, "webSearchRequests": 0, "costUSD": 0.023218, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 260, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4301, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-bug-fix-tdd / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 23m 3s; budget used: 7.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 85026`; `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1603,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.07381494418004783`; `facts.toolUse.totalToolCalls: 36`; `facts.timing.timeToFirstArtifactMs: 85026`; `facts.outcome.durationMs: 1382590` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14934 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31274 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.58 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 66590 + 9498 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 8 → 6 | `continuity.steps` |\n| Gate rejections / redrives | 1 / 0 | `continuity.steps` |\n| Median step wall-clock | 166s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.63 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 14% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 85026`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1603,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.07381494418004783`\n- `facts.toolUse.totalToolCalls: 36`\n- `facts.timing.timeToFirstArtifactMs: 85026`\n- `facts.outcome.durationMs: 1382590`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 85026`; `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1603,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.07381494418004783`; `facts.toolUse.totalToolCalls: 36`; `facts.timing.timeToFirstArtifactMs: 85026`; `facts.outcome.durationMs: 1382590`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nRead all five final deliverables and source/test fixtures; compared final hashes against the provisional review and verified exact red/green receipts and edit order. The final review only removes its invalid empty-findings row. No extra test execution or model retry.\n\nPrimary artifact: 2246 UTF-8 bytes.\n\n| Axis | Score | Exact declared description |\n|---|---:|---|\n| Repro-first discipline | 9 | A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim. |\n| Fix minimality | 9 | The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn. |\n| Honest verification | 7 | Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries. |\n\n**Advisory mean: 8.33/10.**\n\nReviewer: codex-inline/current-session. Fixed score remains **8.5/10**, outcome **pass**.\n\nA real red run precedes the one-character boundary fix; a fresh run then passes all four tests, preserving the original assertions. Notes omit the modified test from Files touched and incorrectly discuss no per-line rounding before an unimplemented summation. The reviewer accepts those claims. Core repair and red-to-green proof are sound.\n\n## Positives\n\n- All 28 deterministic checks passed and the actual craftbook reached terminal completion (outcome.success; sniff.latest.score: 28/28; craftbook.reachedTerminal). This pass demonstrates a completed repair workflow, unlike the earlier dev-craftbook-routing selection-only pass.\n- The boundary regression test was appended at 21:12:37.379Z and the real suite failed at 21:12:47.839Z with 80 !== 68. The source fix followed at 21:19:23.016Z, so this is genuine reproduction before repair (toolUse.byTool.append_to_file: 1, replace_in_file: 1, run_package_script: 2; advisory-execution-audit.json).\n- The entire pricing source differs from the seed only by > becoming >= at the threshold comparison. All original tests are retained, the new boundary test persists, and package.json plus the bug report are unchanged. A fresh run at 21:22:04.783Z passed all four tests with the exact output quoted in validation.md (sniff.latest.score; test receipts and fixture hashes).\n\n## Negatives\n\n- The primary notes and final review list src/pricing.js as the only touched file, omitting the appended test. Their risk discussion claims no per-line rounding before summation, but lineTotal already returns round2 of the discounted subtotal and no summation layer exists in the fixture (toolUse.byTool.append_to_file: 1; captured source and final documents). The reviewer did not correct these claims.\n- The fix notes asserted the new test passes at 21:20:58.131Z, before the first recorded green invocation at 21:22:04.783Z. The later validation genuinely confirms the result, so the terminal code outcome is sound; the initial claim was premature (toolUse.byTool.run_package_script: 2; complete zero-gap event recording). A separate revert test was not performed and is not required to infer the regression guard from the red run.\n- A diagnosis write was refused for source line numbers and the derived factor 0.85 despite preceding reads; a cited rewrite was accepted. The reviewer also needed to replace an empty-findings table row whose severity was “(none)” (toolUse.byTool.write_artifact: 7; advance_task_step: 3; recorded refusals). These repairs added overhead without changing the correct source fix.\n- Eight created roles and 36 tool calls took 23m03s for one production-character edit and one added test (team.totalGezelsCreated; toolUse.totalToolCalls; outcome.durationMs). A clean final unit suite does not validate a browser workflow or an unimplemented aggregation layer.\n\n## Tactical experiment — not applied\n\nIn a future version of ../gilde/data/gezel-templates/re/reviewer/versions/1.4.1/about.md, add: “Reconcile Files touched against actual source and test changes. Derive risk statements from the current code; do not invent adjacent layers. Distinguish a command receipt from an expected result or proposed verification step.” Test on bug fixes and unrelated data-transformation reviews, measuring inaccurate change inventories and unsupported risks.\n\n## Strategic experiment — not applied\n\nIn packages/service/src/chat/tool-evidence-replay.ts, carry compact mutation and command receipts across role handoffs: affected paths, before/after hashes, command, timestamp and exit status. Let the reviewer cross-check prose against those receipts instead of treating summaries as independent evidence. Keep syntax-only and semantic validation receipts distinct. Measure receipt-backed claims across source repair, spreadsheet and document workflows before adoption.\n\n## Performance and fit\n\nElapsed 23.04 minutes; 9498 recorded generated tokens; mean decode 8.58 tokens/sec, prefill 388.52 tokens/sec. Peak VRAM 10638/12227 MiB, process RSS 14934 MiB, whole-system RAM 31274/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). One native 0.1.57 CUDA launch, 65536 context, f16 KV, one slot, 4096 reasoning budget. The planner places dense FFN from 39/64 layers on GPU and 25 on CPU. This is mixed residency, not full GPU residency; shared GPU memory and paging are not separately measured. No explicit CUDA OOM, context overflow, endpoint failure or unexpected native exit was found.\n\n## Generalization and next experiment\n\nThe refactor trial also produced correct code but overstated an intermediate test run; the codemod Reviewer misstated a tool name. This TDD trial has strong real execution proof despite weaker surrounding prose. A reusable receipt handoff addresses all three patterns; special-casing threshold inputs or changing gates would not.\n\nNo tuning, prompts, gates, source files, model weights or fixed scores were changed. Preserve this campaign as the baseline for any later receipt/provenance experiment.\n\n## Last three same-scenario trials\n\n- [craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/facts.json): fail, fixed 5.5, 2653.8 seconds, 114.86 recorded tokens/sec. repair-exhausted (score plateau): 6 completed repairs with the scenario score frozen at 26 while the failing detail kept changing; latest checked file workspace output. Last failure: task sourced from craftbook bug-fix-tdd has not reached a terminal step\n- [craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/facts.json): fail, fixed 5.6, 1269.3 seconds, 119.57 recorded tokens/sec. craftbook-bug-fix-tdd stale no-write repair loop: tasks/1/review.md had unchanged content (729 bytes) for 122s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 27/28: tasks/1/review.md is missing required content: explicit reviewer verdict — nothing in its 729 bytes matches /Verdict:\\s*(?:PASS|REVISE)/. Add that content.\n- [craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-07-37-017Z-ygbs/facts.json): fail, fixed 5.6, 1727.8 seconds, 121.69 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on workspace output; missing signals stayed unchanged: task sourced from craftbook bug-fix-tdd has not reached a terminal step. Last failure: task sourced from craftbook bug-fix-tdd has not reached a terminal step\n\nEarlier outcomes: 0/3 passed. Model/native/source identities differ, so these are historical context, not a controlled native-version comparison. Full reasons are preserved in historical-comparison.json.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json and the execution/selection audit beside this report.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-find-vs-create", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-find-vs-create", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T21:37:06.557Z", "finishedAt": "2026-11-04T21:39:01.829Z", "durationMs": 115272, "success": true, "reason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.04615682155714806 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 115272 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" }, "facts": { "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 18, "peakRssMb": 9856, "peakCpuPercent": 1201.6, "firstAtMs": 0, "lastAtMs": 100360 }, "gpu": { "available": true, "sampleCount": 18, "peakUtilPercent": 99, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 26997, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11882, "totalOutputTokens": 468, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T21:38:54.680Z" } }, "source": "engine-log" }, "derived": { "durationMs": 101784, "meanTokensPerSec": 8.12, "genTokensPerSec": 8.12, "genTokens": 468, "promptTokensPerSec": 617.06, "peakRssMb": 9856, "peakGpuUtilPercent": 99, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shop-site-pages/1", "stepId": "scope", "activatedAt": "2026-11-04T21:38:52.191Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T21:37:12.308Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T21:37:12.205Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "rosalind": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1510467529296875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1510467529296875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/whats-new/1.26234", "knowledge://bendyline/handboek/craftbook/accessibility-retrofit", "knowledge://bendyline/handboek/craftbook/podcast-chaptering" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "durationMs": 115272, "timeoutMs": 2497399, "budgetUsedFraction": 0.04615682155714806 }, "timing": { "startedAt": "2026-11-04T21:37:06.557Z", "finishedAt": "2026-11-04T21:39:01.829Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 45301, "firstTurnTtftMs": 26617, "timeToFirstToolCallMs": 97215 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "suggest_work": 1, "start_work": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/shop-site-pages/pages/index.html", "finalBytes": 168, "growth": [] }, { "path": "workspace/shop-site-pages/pages/shipping.html", "finalBytes": 165, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 7736, "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } ], "latest": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T21:37:06.561Z] [trial] id=craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1 scenario=craftbook-find-vs-create model=underdog-saluki-27b-iq2-mix", "[2026-11-04T21:37:06.561Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-find-vs-create-bMQwir", "[2026-11-04T21:37:06.561Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T21:37:06.561Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "[2026-11-04T21:37:06.561Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T21:37:06.564Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T21:37:06.567Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T21:37:06.573Z] [trial] throughput-scaled ceiling: 20m → 42m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T21:37:06.573Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-find-vs-create-bMQwir", "[2026-11-04T21:37:12.286Z] [trial] daemon spawned pid=43404 port=https://127.0.0.1:65492", "[2026-11-04T21:37:13.712Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T21:37:13.728Z] [trial] running scenario setup hook", "[2026-11-04T21:37:13.728Z] [trial] meester=rosalind", "[2026-11-04T21:37:14.034Z] [trial] scenario setup complete", "[2026-11-04T21:37:14.036Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T21:37:14.118Z] [poll] initial digests hard=4dea43c80e928cc9 soft=91a15da058800b01", "[2026-11-04T21:37:14.384Z] [poll] hard-progress: 4dea43c80e928cc9 → 6d578e98593af760 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/3B)", "[2026-11-04T21:38:48.400Z] [poll] hard-progress: 6d578e98593af760 → 37b81af555e12748 after 94s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=153,stream=455) sniff=1/3B)", "[2026-11-04T21:38:54.201Z] [poll] terminal: success=true reason=task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "[2026-11-04T21:38:54.705Z] [capture] dumped 1 session(s)", "[2026-11-04T21:38:54.708Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-04T21:38:54.768Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-04T21:39:01.828Z] [trial] daemon shut down", "[2026-11-04T21:39:01.837Z] [trial] PASS duration=115272ms reason=task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored" ] }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 9856, "peakCpuPercent": 1201.6, "firstAtMs": 0, "lastAtMs": 100360 }, "gpu": { "available": true, "sampleCount": 18, "peakUtilPercent": 99, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 26997, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11882, "totalOutputTokens": 468, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T21:38:54.680Z" } }, "source": "engine-log" }, "derived": { "durationMs": 101784, "meanTokensPerSec": 8.12, "genTokensPerSec": 8.12, "genTokens": 468, "promptTokensPerSec": 617.06, "peakRssMb": 9856, "peakGpuUtilPercent": 99, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T21:37:06.557Z", "finishedAt": "2026-11-04T21:39:01.829Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 45301, "firstTurnTtftMs": 26617, "timeToFirstToolCallMs": 97215 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shop-site-pages/1", "stepId": "scope", "activatedAt": "2026-11-04T21:38:52.191Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T21:37:12.308Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T21:37:12.205Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "rosalind": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1510467529296875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1510467529296875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "scenarioId": "craftbook-find-vs-create", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "artifactPath": null, "artifactBytes": 0, "artifactSha256": null, "axisSource": "evals/src/craftbooks/authoring/find-vs-create.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "Selection-only scenario: the model found and invoked the bundled seo-meta-pack recipe in two calls, without authoring a new recipe. No model-produced deliverable is required or present. The two HTML pages are seeded fixtures and are excluded from qualitative judging. This pass does not demonstrate completed page metadata.", "reviewBasis": "Read the selection scenario contract and both recorded tool calls; confirmed the start receipt creates shop-site-pages/1 with scope active and later steps pending. Seeded pages are not model output; no qualitative axes were invented.", "deliverables": [] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:26.657Z", "reviewId": "035", "cellId": "r1-saluki-cuda-complex-work-craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "scenarioId": "craftbook-find-vs-create", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-find-vs-create", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "a30da9fc70965aa4f27a319502ada2a719deee5d346b1f91e347b90887d0f476", "evidenceSha256": "ab916625d1e1cad681d30d4578a5d7d45b73d32aa284d63c3d430049cf9b373f", "artifactFiles": [], "contextFiles": [], "scoreAxes": {}, "justification": "No deliverable to score: artifacts is empty, declaredAxes is empty, and missingDeclaredPaths is empty. The evidence has no transcript or tool-call log, so it doesn't show whether suggest_craftbook or invoke_craftbook was called, or whether any page metadata was written. The cause of the missing output can't be determined.", "limitations": "Source-only review with no runtime execution. Executiondisposition says \"runner-terminal\", but no runner outcome or error detail is shown. Partial work can't be judged as success or failure. No numeric scores are given. Any explanation for the missing output would be speculation.", "meanScore": null, "durationMs": 14237, "costUsd": 0.009202, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1482, "cache_read_input_tokens": 700, "output_tokens": 320, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1482, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 320, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1482, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1482 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 320, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1482, "webSearchRequests": 0, "costUSD": 0.009202, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1482, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-find-vs-create / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.8 / 10** (band: ship-ready)\n**Outcome: success** (duration: 1m 55s; budget used: 4.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 5 / 10 | The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.04615682155714806`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 115272` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **8.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9856 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 99% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26997 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.12 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 11882 + 468 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 50% of its measured signals.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.04615682155714806`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 115272`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.04615682155714806`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 115272`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nRead the selection scenario contract and both recorded tool calls; confirmed the start receipt creates shop-site-pages/1 with scope active and later steps pending. Seeded pages are not model output; no qualitative axes were invented.\n\n**noArtifact** — no model-produced primary deliverable. The seeded HTML fixtures are excluded from judging.\n\nReviewer: codex-inline/current-session. Fixed score remains **8.8/10**, outcome **pass**.\n\nSelection-only scenario: the model found and invoked the bundled seo-meta-pack recipe in two calls, without authoring a new recipe. No model-produced deliverable is required or present. The two HTML pages are seeded fixtures and are excluded from qualitative judging. This pass does not demonstrate completed page metadata.\n\n## Positives\n\n- The model selected the bundled seo-meta-pack recipe and created its task without authoring any local recipe (outcome.success; outcome.reason). The actual tools were suggest_work followed by start_work with the exact returned option craftbook:seo-meta-pack (toolUse.byTool.suggest_work: 1; start_work: 1).\n- The suggestion returned the fitting recipe first, and task creation succeeded in two calls with no recorded red flags or missing roles (toolUse.totalToolCalls: 2; toolUse.redFlags: []; team.missingExpectedRoles: []). No capability retry was needed.\n\n## Negatives\n\n- The trial intentionally stops at recipe selection/task creation. Its start receipt has scope active and build/evaluate/finish pending; it does not establish that titles, descriptions or social-card metadata were generated (outcome.reason; advisory-selection-audit.json).\n- facts.artifacts.htmlFiles contains the two seeded pages, and sniff.latest remains the pre-terminal 1/2 snapshot even though the final task-creation check passed. The fixed quality axis uses that stale snapshot and produces 8.8 overall. This is a reporting/snapshot limitation, not failed selection or poor model-authored HTML (sniff.latest; outcome.success; score.json). No fixed score is changed.\n- The run took 1m55s for 468 recorded generated tokens, including startup and task dispatch (outcome.durationMs; perf.derived.genTokens). The short trial’s smaller RSS peak should not be generalized to long multi-role work.\n\n## Tactical experiment — not applied\n\nFor a future version of ../gilde/data/gezel-templates/me/meester/versions/1.4.1/about.md, explicitly tie the user-facing completion statement to the returned start receipt: active/pending means work started, and only a completed result means the output is finished. This trial’s selection already works in the minimum two calls; measure completion-claim accuracy rather than adding needless selection steps across development and productivity recipes.\n\n## Strategic experiment — not applied\n\nIn evals/src/craftbooks/authoring/find-vs-create.ts and its shared finishAuthoringPoll reporting path, emit the terminal selection evidence alongside the terminal result so aggregate reporting cannot pair success with a stale pre-terminal sniff. Keep selection-only probes explicitly labeled in reports and exclude seeded fixtures from advisory artifact scoring. Validate the reporting change on the unrelated dev-craftbook-routing probe too; do not alter this frozen score or current gates.\n\n## Performance and fit\n\nElapsed 1.92 minutes; 468 recorded generated tokens; mean decode 8.12 tokens/sec, prefill 617.06 tokens/sec. Peak VRAM 10628/12227 MiB, process RSS 9856 MiB, whole-system RAM 26997/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). One native 0.1.57 CUDA launch, 65536 context, f16 KV, one slot, 4096 reasoning budget. The planner places dense FFN from 39/64 layers on GPU and 25 on CPU. This is mixed residency, not full GPU residency; shared GPU memory and paging are not separately measured. No explicit CUDA OOM, context overflow, endpoint failure or unexpected native exit was found.\n\n## Generalization and next experiment\n\nThis resembles the earlier dev-craftbook-routing pass: both establish correct recipe choice, not completion of the chosen task. By contrast, bug-fix-tdd reached terminal workflow completion and has real red/green execution evidence. Keeping these outcome scopes distinct prevents overstating diverse task competence.\n\nNo tuning, prompts, gates, source files, model weights or fixed scores were changed. Preserve this campaign as the baseline for any later receipt/provenance experiment.\n\n## Last three same-scenario trials\n\n- [craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/facts.json): pass, fixed 8.8, 49.1 seconds, 108.41 recorded tokens/sec. task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored\n- [craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt/facts.json): pass, fixed 8.8, 48.2 seconds, 104.14 recorded tokens/sec. task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored\n- [craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-41-57-643Z-c248/facts.json): pass, fixed 8.8, 97.6 seconds, 104.09 recorded tokens/sec. task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored\n\nEarlier outcomes: 3/3 passed. Model/native/source identities differ, so these are historical context, not a controlled native-version comparison. Full reasons are preserved in historical-comparison.json.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json and the execution/selection audit beside this report.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-edit-midtask", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-edit-midtask", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T21:41:20.385Z", "finishedAt": "2026-11-04T23:00:17.918Z", "durationMs": 4737533, "success": true, "reason": "diagnosed the unwinnable gate, rewrote the failing step prompt, grew the recipe with a verification step, and completed the task", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2317014 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-edit-midtask", "bytes": 571, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.8431053681279382 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 70 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2317014 }, { "source": "facts", "path": "outcome.durationMs", "value": 4737533 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 794, "peakRssMb": 15294, "peakCpuPercent": 1231.1, "firstAtMs": 0, "lastAtMs": 4726795 }, "gpu": { "available": true, "sampleCount": 794, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 794, "peakUsedMb": 31721, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 115252, "totalOutputTokens": 35469, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 162324, "todayTokensCached": 149008, "todayTokensOut": 2550, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 162324, "totalTokensCached": 149008, "totalTokensOut": 2550, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 13 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8012, "cachedInputTokens": 7816, "outputTokens": 93, "cost": 0, "durationMs": 358275, "outputTokensPerSec": 8.560219469139849, "at": "2026-11-04T23:00:12.645Z" }, "lastUpdated": "2026-11-04T23:00:12.645Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T23:00:14.428Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4728303, "meanTokensPerSec": 8.35, "genTokensPerSec": 8.35, "genTokens": 35469, "promptTokensPerSec": 362.79, "peakRssMb": 15294, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 162324, "output": 2550 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "quarterly-brief/1", "stepId": "draft", "activatedAt": "2026-11-04T21:41:27.248Z", "completedAt": "2026-11-04T22:34:58.825Z", "ms": 3211577, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "quarterly-brief/1", "stepId": "polish", "activatedAt": "2026-11-04T22:34:58.826Z", "completedAt": "2026-11-04T22:53:06.837Z", "ms": 1088011, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "3d5e0e40-3a45-4aca-94d1-7f3971e157d0" ] }, { "ref": "quarterly-brief/1", "stepId": "verify", "activatedAt": "2026-11-04T22:53:06.839Z", "completedAt": "2026-11-04T23:00:12.569Z", "ms": 425730, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "3d5e0e40-3a45-4aca-94d1-7f3971e157d0" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T21:41:26.096Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T21:41:26.029Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1088011 }, "sessions": { "total": 5, "taskScoped": 1, "byGezel": { "fitri": 1, "jyoti": 1, "reza": 2, "reza-2": 1 }, "perTask": { "quarterly-brief/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.2, "maxMessages": 9, "maxContextFill": 0.197540283203125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.197540283203125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "diagnosed the unwinnable gate, rewrote the failing step prompt, grew the recipe with a verification step, and completed the task", "durationMs": 4737533, "timeoutMs": 5619147, "budgetUsedFraction": 0.8431053681279382 }, "timing": { "startedAt": "2026-11-04T21:41:20.385Z", "finishedAt": "2026-11-04T23:00:17.918Z", "timeToFirstArtifactMs": 2317014, "timeToLastArtifactWriteMs": 4364372, "timeToFirstTokenMs": 45253, "firstTurnTtftMs": 35549, "timeToFirstToolCallMs": 53667 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 70, "byTool": { "get_task": 4, "read_task_notes": 4, "craftbook_read": 1, "read_file": 10, "list_dir": 2, "craftbook_write": 1, "list_project_gezels": 1, "message_gezel": 3, "list_artifacts": 2, "grep_files": 12, "delegate_developer": 4, "write_task_note": 5, "find_files": 1, "search": 3, "stat": 4, "advance_task_step": 3, "map_repo": 3, "list_projects": 1, "ensure_gezel": 1, "write_file": 5 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 3 }, "sniff": { "progression": [ { "atMs": 7197, "filePath": "craftbook-edit-midtask", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" }, { "atMs": 748803, "filePath": "craftbook-edit-midtask", "bytes": 0, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 2322495, "filePath": "craftbook-edit-midtask", "bytes": 776, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3174819, "filePath": "craftbook-edit-midtask", "bytes": 571, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3264264, "filePath": "craftbook-edit-midtask", "bytes": 776, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 4144228, "filePath": "craftbook-edit-midtask", "bytes": 624, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 4370164, "filePath": "craftbook-edit-midtask", "bytes": 571, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 4733510, "filePath": "craftbook-edit-midtask", "bytes": 571, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-edit-midtask", "bytes": 571, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T21:41:20.390Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "[2026-11-04T21:41:20.390Z] [trial] id=craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c scenario=craftbook-edit-midtask model=underdog-saluki-27b-iq2-mix", "[2026-11-04T21:41:20.390Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-edit-midtask-Lh3c09", "[2026-11-04T21:41:20.390Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T21:41:20.390Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T21:41:20.392Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T21:41:20.396Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T21:41:20.402Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-edit-midtask-Lh3c09", "[2026-11-04T21:41:20.402Z] [trial] throughput-scaled ceiling: 45m → 94m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T21:41:26.133Z] [trial] daemon spawned pid=16788 port=https://127.0.0.1:54796", "[2026-11-04T21:41:27.008Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T21:41:27.024Z] [trial] running scenario setup hook", "[2026-11-04T21:41:27.024Z] [trial] meester=tamsin", "[2026-11-04T21:41:27.493Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T21:41:27.491Z] [trial] scenario setup complete", "[2026-11-04T21:41:27.562Z] [poll] initial digests hard=9c651cf917e44866 soft=039d81658cd92c16", "[2026-11-04T21:41:27.734Z] [poll] hard-progress: 9c651cf917e44866 → 23fee151457bc58d after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T21:42:14.613Z] [poll] hard-progress: 23fee151457bc58d → 1f01fcdc99110634 after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=31,stream=54) sniff=1/0B)", "[2026-11-04T21:44:53.998Z] [poll] hard-progress: 1f01fcdc99110634 → edbe1e0b744bb18a after 159s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=456,stream=1322) sniff=1/0B)", "[2026-11-04T21:47:21.947Z] [poll] hard-progress: edbe1e0b744bb18a → 6d62dff61e14b9f9 after 148s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=864,stream=2537) sniff=1/0B)", "[2026-11-04T21:53:43.319Z] [poll] hard-progress: 6d62dff61e14b9f9 → b03d4f761c66e124 after 381s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=1890,stream=6084) sniff=1/0B)", "[2026-11-04T21:53:49.325Z] [poll] hard-progress: b03d4f761c66e124 → b3cb28e57b756a4d after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=1897,stream=6106) sniff=3/0B)", "[2026-11-04T21:55:54.575Z] [poll] hard-progress: b3cb28e57b756a4d → db63f797dc2951d0 after 125s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=2229,stream=7098) sniff=3/0B)", "[2026-11-04T21:57:24.108Z] [poll] hard-progress: db63f797dc2951d0 → 63e6b9d04b7c1224 after 90s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=10,slot=2462,stream=7928) sniff=3/0B)", "[2026-11-04T21:58:17.782Z] [poll] hard-progress: 63e6b9d04b7c1224 → d3bb41b221dcf3a4 after 54s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=3,tools=12,slot=2563,stream=8208) sniff=3/0B)" ] }, "metrics": { "process": { "sampleCount": 794, "peakRssMb": 15294, "peakCpuPercent": 1231.1, "firstAtMs": 0, "lastAtMs": 4726795 }, "gpu": { "available": true, "sampleCount": 794, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 794, "peakUsedMb": 31721, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 115252, "totalOutputTokens": 35469, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 13, "todayTokensIn": 162324, "todayTokensCached": 149008, "todayTokensOut": 2550, "todayCost": 0, "totalTurns": 13, "totalTokensIn": 162324, "totalTokensCached": 149008, "totalTokensOut": 2550, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 13 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8012, "cachedInputTokens": 7816, "outputTokens": 93, "cost": 0, "durationMs": 358275, "outputTokensPerSec": 8.560219469139849, "at": "2026-11-04T23:00:12.645Z" }, "lastUpdated": "2026-11-04T23:00:12.645Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-04T23:00:14.428Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4728303, "meanTokensPerSec": 8.35, "genTokensPerSec": 8.35, "genTokens": 35469, "promptTokensPerSec": 362.79, "peakRssMb": 15294, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 162324, "output": 2550 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T21:41:20.385Z", "finishedAt": "2026-11-04T23:00:17.918Z", "timeToFirstArtifactMs": 2317014, "timeToLastArtifactWriteMs": 4364372, "timeToFirstTokenMs": 45253, "firstTurnTtftMs": 35549, "timeToFirstToolCallMs": 53667 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "quarterly-brief/1", "stepId": "draft", "activatedAt": "2026-11-04T21:41:27.248Z", "completedAt": "2026-11-04T22:34:58.825Z", "ms": 3211577, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "quarterly-brief/1", "stepId": "polish", "activatedAt": "2026-11-04T22:34:58.826Z", "completedAt": "2026-11-04T22:53:06.837Z", "ms": 1088011, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "3d5e0e40-3a45-4aca-94d1-7f3971e157d0" ] }, { "ref": "quarterly-brief/1", "stepId": "verify", "activatedAt": "2026-11-04T22:53:06.839Z", "completedAt": "2026-11-04T23:00:12.569Z", "ms": 425730, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "3d5e0e40-3a45-4aca-94d1-7f3971e157d0" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T21:41:26.096Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T21:41:26.029Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1088011 }, "sessions": { "total": 5, "taskScoped": 1, "byGezel": { "fitri": 1, "jyoti": 1, "reza": 2, "reza-2": 1 }, "perTask": { "quarterly-brief/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.2, "maxMessages": 9, "maxContextFill": 0.197540283203125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.197540283203125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "scenarioId": "craftbook-edit-midtask", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/workspace/quarterly-brief/out/summary.md", "artifactBytes": 571, "artifactSha256": "4d8064f3dbe41999f9a3cc454938f72c55263e6a6451df3ab290aa56319222e9", "axisSource": "evals/src/craftbooks/authoring/edit-midtask.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. Final brief faithfully reproduces the supplied Q2 facts, and the repaired three-step task completes with the original gate intact. Execution took five writes, including fabricated content and a stale overwrite of a correct draft. Setup also created two distinct authoring workers, splitting ownership from repair.", "reviewBasis": "Read the final brief against source notes, final task craftbook/state, all five write attempts and task transitions. Inspected helper source and setup logs to confirm duplicate-worker creation. Retained the prior incorrect checkpoint as evidence; no manual repair or capability retry." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:31.058Z", "reviewId": "036", "cellId": "r1-saluki-cuda-complex-work-craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "scenarioId": "craftbook-edit-midtask", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-edit-midtask", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "2c10e5917868a096fa03fc79dc87c66e94f932e58bb55ece42bd215a0c4ac920", "evidenceSha256": "6e739cc20031c3118e4b65d49816f312b8694f94164bd4f4a7ac3cd80e2bb46b", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/workspace/quarterly-brief/out/summary.md", "bytes": 571, "sha256": "4d8064f3dbe41999f9a3cc454938f72c55263e6a6451df3ab290aa56319222e9", "shownBytes": 571, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c\\workspace\\quarterly-brief\\data\\notes.txt", "bytes": 544, "sha256": "949c911a4bbdf1083c68bbe94d539edda30128a353f4028a07a65bb717752a47", "shownBytes": 544, "truncated": false } ], "scoreAxes": {}, "justification": "No numeric axes declared. Evidence shows only the final summary.md, which faithfully matches notes.txt and adds a Source Register section, plausibly what the gate demanded. No craftbook reads/edits, added verification step, gate, or task completion are visible, so the fix and completion cannot be confirmed.", "limitations": "The craftbook, tool-call transcript, gate definition and task status were not provided. Whether the recipe was edited, a verification step added, or the task completed is unobserved; the output file alone doesn't prove the intended process was followed.", "meanScore": null, "durationMs": 14550, "costUsd": 0.012038, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2246, "cache_read_input_tokens": 700, "output_tokens": 298, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2246, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 298, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2246, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2246 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 298, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2246, "webSearchRequests": 0, "costUSD": 0.012038, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2246, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-edit-midtask / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 78m 58s; budget used: 84.3%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 2317014`; `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":571,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.8431053681279382`; `facts.toolUse.totalToolCalls: 70`; `facts.timing.timeToFirstArtifactMs: 2317014`; `facts.outcome.durationMs: 4737533` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15294 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31721 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.35 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 115252 + 35469 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1088s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.20 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 20% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 2317014`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":571,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.8431053681279382`\n- `facts.toolUse.totalToolCalls: 70`\n- `facts.timing.timeToFirstArtifactMs: 2317014`\n- `facts.outcome.durationMs: 4737533`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 2317014`; `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":571,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.8431053681279382`; `facts.toolUse.totalToolCalls: 70`; `facts.timing.timeToFirstArtifactMs: 2317014`; `facts.outcome.durationMs: 4737533`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nRead the final brief against source notes, final task craftbook/state, all five write attempts and task transitions. Inspected helper source and setup logs to confirm duplicate-worker creation. Retained the prior incorrect checkpoint as evidence; no manual repair or capability retry.\n\nNo qualitative axes are declared. This is a prose-only review of the 571-byte final brief, with codex-inline/current-session metadata. Fixed result remains **pass, 8.5/10**.\n\nNo declared axes. Final brief faithfully reproduces the supplied Q2 facts, and the repaired three-step task completes with the original gate intact. Execution took five writes, including fabricated content and a stale overwrite of a correct draft. Setup also created two distinct authoring workers, splitting ownership from repair.\n\n## Positives\n\n- The final task is complete, the seeded polish prompt is rewritten, a terminal verify step was added, and the original polish gate checks/rejection target/maxAttempts are preserved (outcome.success; sniff.latest.score: 4/4; captured final state and advisory-write-audit.json). The model performed the requested live craftbook edit rather than replacing the task.\n- The final 571-byte brief preserves the supplied wins, misses and numbers, including revenue 148k, 12% QoQ growth, 41% margin, 210k sessions, 2.4% conversion and 1.1% refunds. It correctly names the source register and omits the earlier invented business facts (sniff.latest.bytes: 571; source notes and final artifact).\n- Reza read both the wrong draft and the actual source, explicitly recognized fabricated onboarding/pricing/churn content, rewrote the brief correctly at 22:50:20.378Z, then advanced the real polish/verify steps. A later Jyoti write retained correct content, and the terminal verify advance succeeded at 23:00:12.644Z (toolUse.byTool.read_file: 10; write_file: 5; advance_task_step: 3).\n\n## Negatives\n\n- The first Jyoti brief invented $4.7M revenue, onboarding and pricing-page launches, app ratings and churn unrelated to the supplied notes. The correct 571-byte draft written at 22:34:12.934Z was then replaced at 22:35:42.178Z by Fitri’s stale 780-byte copy of that invented draft. This regression was later repaired, so it is an execution failure history, not the final artifact verdict (toolUse.byTool.write_file: 5; advisory-write-audit.json).\n- The scenario setup calls ensureAuthoringWorker twice. That helper always tries createGezel first and only reuses on error, so the logs show reza created at 21:41:27.240Z and reza-2 at 21:41:27.335Z. The task belongs to reza while the explicit repair kickoff goes to reza-2. This contradicts the setup’s intended one-actor loop and creates a concrete harness contribution to coordination overhead (team.totalGezelsCreated: 8; setup log; evals/src/craftbooks/authoring/helpers.ts and edit-midtask.ts).\n- Reza-2 claimed the draft was complete and the file existed at 21:57:18.521Z, but the first write was not until 22:19:57.407Z; multiple later reads and searches still found nothing. At 22:55:20.016Z a message claimed task completion and 780 bytes even though the latest file was 571 bytes and verify was not complete. Terminal completion only occurred at 23:00:12.644Z (timing.timeToFirstArtifactMs: 2317014; toolUse.byTool.message_gezel: 3; recorded chronology).\n- Four developer delegations and several overlapping workers kept requesting or rewriting an already produced file; one more delegation was accepted just before terminal completion. Seventy calls and 35469 generated tokens took 78m58s for a small brief and recipe repair (toolUse.totalToolCalls; byTool.delegate_developer: 4; perf.derived.genTokens; outcome.durationMs).\n- The added verification step checks byte length and heading presence, not factual correspondence. Fitri initially accepted the invented draft on those checks. The final source correction came from reading the notes, so the four-check workflow pass alone would not have established accurate content (sniff.latest.score; captured craftbook prompts and write/read sequence).\n\n## Tactical experiment — not applied\n\nIn AUTHORING_WORKER_ABOUT in evals/src/craftbooks/authoring/helpers.ts, add a general execution rule: “A delegated write is pending until a successful write receipt or fresh read confirms it. Before sending a completion or verification claim, reread the current task status and artifact; stop requesting work already completed.” Measure unsupported existence/completion claims and redundant delegations on live recipe repair and unrelated report-writing tasks. Do not include scenario-specific headings or expected facts.\n\n## Strategic experiments — not applied\n\nIn evals/src/craftbooks/authoring/edit-midtask.ts, resolve the authoring worker once and reuse that returned id for both task assignment and kickoff; alternatively make ensureAuthoringWorker genuinely find-or-create in helpers.ts. The two observed ids and the helper implementation establish this setup defect, independently of model output. Separately, in packages/service/src/chat/manager.ts, attach the read version/hash to delegated artifact edits and detect stale full-file overwrites before they replace a newer version. Measure concurrent-writer regressions on this flow and the unrelated deep-security-review/document workflows. Preserve all attempted writes for inspection. Neither experiment has been applied to this frozen campaign.\n\n## Performance and fit\n\nElapsed 78.96 minutes; 35469 recorded generated tokens; mean decode 8.35 tokens/sec, prefill 362.79 tokens/sec. Peak VRAM 10646/12227 MiB; process RSS 15294 MiB; whole-system RAM 31721/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). The whole-system sample leaves only 144 MiB below the reported total; it is not solely model RAM. One native 0.1.57 CUDA launch, 65536-token context, f16 KV, one slot and 4096 reasoning budget. Dense FFN from 39/64 layers is placed on GPU and 25 on CPU. This is mixed residency, not a fully GPU-resident model. Shared GPU memory and paging are not separately measured. No explicit OOM, context overflow, endpoint failure or unexpected native exit was found.\n\n## Generalization and next experiment\n\nDeep-security-review also suffered a second writer replacing stronger content, whereas this trial ultimately repaired the regression. Bug-fix-tdd and refactor-module show that correct source outputs can coexist with inaccurate execution claims. Validate a single-owner, version-aware handoff plus fresh completion receipts across code, document and workflow-editing tasks. This targets observed coordination/provenance failures rather than memorizing a gate or recommending a larger model.\n\n## Last three same-scenario trials\n\n- [craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/facts.json): fail, fixed 4.5, 443.4 seconds, 111.83 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on task quarterly-brief/1 craftbook; missing signals stayed unchanged: task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it, task quarterly-brief/1's craftbook still has 2 steps — add a verification step at the end of the recipe. Last failure: task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to \n- [craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig/facts.json): pass, fixed 8.5, 172.5 seconds, 106.42 recorded tokens/sec. diagnosed the unwinnable gate, rewrote the failing step prompt, grew the recipe with a verification step, and completed the task\n- [craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-45-19-083Z-o95i/facts.json): pass, fixed 8.5, 156.9 seconds, 105.03 recorded tokens/sec. diagnosed the unwinnable gate, rewrote the failing step prompt, grew the recipe with a verification step, and completed the task\n\nEarlier outcomes: 2/3 passed. Different model/native/source identities make these historical context, not a controlled native-version comparison. No current tuning, prompt, gate, source, weight or score was changed.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json and advisory-write-audit.json. The earlier provisional review remains preserved as evidence of the stale draft.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-invoice-run", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-invoice-run", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-04T23:03:24.958Z", "finishedAt": "2026-11-05T00:02:09.433Z", "durationMs": 3524475, "success": false, "reason": "retry loop (fast-path): sniff \"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\" stuck for 15m — 3 re-writes (28 tool calls) without sniff movement. Artifact produced but never reached success (retry-loop nudge was sent and ignored).", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-invoice-run", "score": 9, "bytes": 783, "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet", "milestones": 6 }, "failureClass": "model", "failureClassRule": "tool-repeat-abort-storm", "failureClassEvidence": "3× read-tool repeat aborts in one trial" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "tool-repeat-abort-storm", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "retry loop (fast-path): sniff \"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\" stuck for 15m — 3 re-writes (28 tool calls) without sniff movement. Artifact produced but never reached success (retry-loop nudge was sent and ignored)." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 363109 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-invoice-run", "bytes": 783, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" } } ] }, "quality": { "score": 9, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 90% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 363109 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-invoice-run", "bytes": 783, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.5645034448648641 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 86 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 363109 }, { "source": "facts", "path": "outcome.durationMs", "value": 3524475 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.5, "band": "needs-tuning" }, "facts": { "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 591, "peakRssMb": 14701, "peakCpuPercent": 1288.1, "firstAtMs": 1, "lastAtMs": 3509723 }, "gpu": { "available": true, "sampleCount": 591, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 591, "peakUsedMb": 30866, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 294488, "totalOutputTokens": 21159, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 152995, "totalTokensCached": 150899, "totalTokensOut": 1602, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 11 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 20888, "cachedInputTokens": 20813, "outputTokens": 26, "cost": 0, "durationMs": 771606, "outputTokensPerSec": 8.587060947867267, "at": "2026-11-04T23:56:52.286Z" }, "lastUpdated": "2026-11-04T23:56:52.286Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T00:02:02.536Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3511230, "meanTokensPerSec": 8.44, "genTokensPerSec": 8.44, "genTokens": 21159, "promptTokensPerSec": 381.48, "peakRssMb": 14701, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 152995, "output": 1602 } } } }, "craftbook": { "craftbookId": "invoice-run", "tasksObserved": 9, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 12, "activationEvents": 5, "completed": 9, "gateApprovals": 8, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "fieldnote-studio-office/1", "stepId": "scope", "activatedAt": "2026-11-04T23:03:32.674Z", "completedAt": "2026-11-04T23:09:28.154Z", "ms": 355480, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "e879ef16-59b5-4b74-a35c-414a5db09a48" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "draft", "activatedAt": "2026-11-04T23:34:03.114Z", "completedAt": "2026-11-04T23:34:03.343Z", "ms": 229, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "fieldnote-studio-office/1", "stepId": "collect", "activatedAt": "2026-11-04T23:34:03.344Z", "completedAt": "2026-11-04T23:25:46.324Z", "ms": null, "gateRejections": 0, "toolCalls": 38, "sessionIds": [ "9cd37441-6869-407b-8c46-8fe20136aa03", "3513f8c6-7208-4bd7-93ae-78110788337a" ] }, { "ref": "fieldnote-studio-office/2", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:09:28.175Z", "completedAt": "2026-11-04T23:13:31.065Z", "ms": 242890, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "dcfc8095-cbdb-4e0b-8aa6-d6f518dea219" ] }, { "ref": "fieldnote-studio-office/3", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:09:28.300Z", "completedAt": "2026-11-04T23:17:42.431Z", "ms": 494131, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "0106f435-1fe5-49a5-917d-cd1fc4a7839f" ] }, { "ref": "fieldnote-studio-office/4", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:09:28.345Z", "completedAt": "2026-11-04T23:21:53.916Z", "ms": 745571, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "c998a70d-fe38-4ed5-bc67-29189a51075a" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "evaluate", "activatedAt": "2026-11-04T23:25:46.325Z", "completedAt": null, "ms": null, "gateRejections": 1, "toolCalls": 10, "sessionIds": [ "e1096cce-84f6-4532-9d36-371a0268b063" ] }, { "ref": "fieldnote-studio-office/5", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:34:03.140Z", "completedAt": "2026-11-04T23:40:43.919Z", "ms": 400779, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "fd53c9ad-2722-47f2-9f8f-450d171772e3" ] }, { "ref": "fieldnote-studio-office/6", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:34:03.197Z", "completedAt": "2026-11-04T23:47:58.781Z", "ms": 835584, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "50de0120-f0c0-4a5a-8383-7683969274cd" ] }, { "ref": "fieldnote-studio-office/7", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:34:03.255Z", "completedAt": "2026-11-04T23:56:34.432Z", "ms": 1351177, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "74f6d168-c8f4-4cea-b8e9-dc0f34ecd34e" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T23:03:30.823Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T23:03:30.713Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 447455 }, "sessions": { "total": 10, "taskScoped": 10, "byGezel": { "chiamaka": 1, "isadora": 6, "nils": 2, "pallavi": 1 }, "perTask": { "fieldnote-studio-office/1": 4, "fieldnote-studio-office/3": 1, "fieldnote-studio-office/6": 1, "fieldnote-studio-office/7": 1, "fieldnote-studio-office/4": 1, "fieldnote-studio-office/2": 1, "fieldnote-studio-office/5": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.8333333333333334, "maxMessages": 8, "maxContextFill": 0.2092437744140625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2092437744140625 }, "fanout": { "hosts": 1, "childrenSpawned": 6, "childrenCompleted": 6, "childrenFailed": 0, "barrierHolds": 2, "barrierReleases": 2, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 3 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 9, "tasksWithReferences": 7, "items": 35, "citations": [ "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "retry loop (fast-path): sniff \"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\" stuck for 15m — 3 re-writes (28 tool calls) without sniff movement. Artifact produced but never reached success (retry-loop nudge was sent and ignored).", "durationMs": 3524475, "timeoutMs": 6243496, "budgetUsedFraction": 0.5645034448648641 }, "timing": { "startedAt": "2026-11-04T23:03:24.958Z", "finishedAt": "2026-11-05T00:02:09.433Z", "timeToFirstArtifactMs": 363109, "timeToLastArtifactWriteMs": 3516366, "timeToFirstTokenMs": 35472, "firstTurnTtftMs": 21493, "timeToFirstToolCallMs": 42086 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Office manager", "Office Administrator", "Designer", "Copywriter", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 86, "byTool": { "read_files": 2, "write_artifact": 4, "write_task_note": 2, "read_task_notes": 7, "read_file": 39, "write_file": 14, "advance_task_step": 9, "read_artifact": 7, "list_dir": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/fieldnote-studio-office/invoices/2026-042.html", "finalBytes": 4433, "growth": [] }, { "path": "workspace/fieldnote-studio-office/invoices/2026-043.html", "finalBytes": 4471, "growth": [] }, { "path": "workspace/fieldnote-studio-office/invoices/2026-044.html", "finalBytes": 4402, "growth": [] } ], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 7956, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 1, "scoreMax": 10, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: clients.json, ledger.json" }, { "atMs": 42674, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 600522, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 851212, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 1106972, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 3, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 1309194, "filePath": "craftbook-invoice-run", "bytes": 802, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 2376380, "filePath": "craftbook-invoice-run", "bytes": 842, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 3184161, "filePath": "craftbook-invoice-run", "bytes": 846, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 3517103, "filePath": "craftbook-invoice-run", "bytes": 783, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" } ], "latest": { "filePath": "craftbook-invoice-run", "bytes": 783, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-04T23:03:24.960Z] [trial] id=craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g scenario=craftbook-invoice-run model=underdog-saluki-27b-iq2-mix", "[2026-11-04T23:03:24.961Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-invoice-run-Jesz9N", "[2026-11-04T23:03:24.961Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-04T23:03:24.961Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "[2026-11-04T23:03:24.961Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-04T23:03:24.963Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-04T23:03:24.967Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-04T23:03:24.971Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-invoice-run-Jesz9N", "[2026-11-04T23:03:24.971Z] [trial] throughput-scaled ceiling: 50m → 104m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-04T23:03:30.807Z] [trial] daemon spawned pid=30436 port=https://127.0.0.1:56213", "[2026-11-04T23:03:32.228Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-04T23:03:32.244Z] [trial] meester=jasper", "[2026-11-04T23:03:32.245Z] [trial] running scenario setup hook", "[2026-11-04T23:03:32.823Z] [trial] scenario setup complete", "[2026-11-04T23:03:32.824Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-04T23:03:32.868Z] [poll] initial digests hard=26c6d9171f1c1655 soft=44097330a6333c8a", "[2026-11-04T23:03:32.952Z] [poll] hard-progress: 26c6d9171f1c1655 → c8653f448756f67b after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T23:03:38.052Z] [poll] hard-progress: c8653f448756f67b → 2c6ed6a9e46abc65 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-04T23:04:07.684Z] [poll] hard-progress: 2c6ed6a9e46abc65 → bb2a1f03b4f4773a after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=24,stream=49) sniff=2/0B)", "[2026-11-04T23:07:28.266Z] [poll] hard-progress: bb2a1f03b4f4773a → 5c38278ee024397a after 201s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=575,stream=2162) sniff=2/0B)", "[2026-11-04T23:08:45.362Z] [poll] hard-progress: 5c38278ee024397a → b14d347f33360afb after 77s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=778,stream=2921) sniff=2/0B)", "[2026-11-04T23:09:33.254Z] [poll] hard-progress: b14d347f33360afb → 15c7bc88c3a7eaa6 after 48s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=801,stream=2990) sniff=2/0B)", "[2026-11-04T23:10:03.132Z] [poll] hard-progress: 15c7bc88c3a7eaa6 → 858408b3dbf3b5ba after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=820,stream=3025) sniff=2/0B)", "[2026-11-04T23:10:09.041Z] [poll] hard-progress: 858408b3dbf3b5ba → 4b6643be5148d271 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=828,stream=3045) sniff=2/0B)", "[2026-11-04T23:10:14.925Z] [poll] hard-progress: 4b6643be5148d271 → b9fadedfbb98de49 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=835,stream=3065) sniff=2/0B)" ] }, "metrics": { "process": { "sampleCount": 591, "peakRssMb": 14701, "peakCpuPercent": 1288.1, "firstAtMs": 1, "lastAtMs": 3509723 }, "gpu": { "available": true, "sampleCount": 591, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 591, "peakUsedMb": 30866, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 294488, "totalOutputTokens": 21159, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 152995, "totalTokensCached": 150899, "totalTokensOut": 1602, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 11 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 20888, "cachedInputTokens": 20813, "outputTokens": 26, "cost": 0, "durationMs": 771606, "outputTokensPerSec": 8.587060947867267, "at": "2026-11-04T23:56:52.286Z" }, "lastUpdated": "2026-11-04T23:56:52.286Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T00:02:02.536Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3511230, "meanTokensPerSec": 8.44, "genTokensPerSec": 8.44, "genTokens": 21159, "promptTokensPerSec": 381.48, "peakRssMb": 14701, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 152995, "output": 1602 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-04T23:03:24.958Z", "finishedAt": "2026-11-05T00:02:09.433Z", "timeToFirstArtifactMs": 363109, "timeToLastArtifactWriteMs": 3516366, "timeToFirstTokenMs": 35472, "firstTurnTtftMs": 21493, "timeToFirstToolCallMs": 42086 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 12, "activationEvents": 5, "completed": 9, "gateApprovals": 8, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "fieldnote-studio-office/1", "stepId": "scope", "activatedAt": "2026-11-04T23:03:32.674Z", "completedAt": "2026-11-04T23:09:28.154Z", "ms": 355480, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "e879ef16-59b5-4b74-a35c-414a5db09a48" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "draft", "activatedAt": "2026-11-04T23:34:03.114Z", "completedAt": "2026-11-04T23:34:03.343Z", "ms": 229, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "fieldnote-studio-office/1", "stepId": "collect", "activatedAt": "2026-11-04T23:34:03.344Z", "completedAt": "2026-11-04T23:25:46.324Z", "ms": null, "gateRejections": 0, "toolCalls": 38, "sessionIds": [ "9cd37441-6869-407b-8c46-8fe20136aa03", "3513f8c6-7208-4bd7-93ae-78110788337a" ] }, { "ref": "fieldnote-studio-office/2", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:09:28.175Z", "completedAt": "2026-11-04T23:13:31.065Z", "ms": 242890, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "dcfc8095-cbdb-4e0b-8aa6-d6f518dea219" ] }, { "ref": "fieldnote-studio-office/3", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:09:28.300Z", "completedAt": "2026-11-04T23:17:42.431Z", "ms": 494131, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "0106f435-1fe5-49a5-917d-cd1fc4a7839f" ] }, { "ref": "fieldnote-studio-office/4", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:09:28.345Z", "completedAt": "2026-11-04T23:21:53.916Z", "ms": 745571, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "c998a70d-fe38-4ed5-bc67-29189a51075a" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "evaluate", "activatedAt": "2026-11-04T23:25:46.325Z", "completedAt": null, "ms": null, "gateRejections": 1, "toolCalls": 10, "sessionIds": [ "e1096cce-84f6-4532-9d36-371a0268b063" ] }, { "ref": "fieldnote-studio-office/5", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:34:03.140Z", "completedAt": "2026-11-04T23:40:43.919Z", "ms": 400779, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "fd53c9ad-2722-47f2-9f8f-450d171772e3" ] }, { "ref": "fieldnote-studio-office/6", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:34:03.197Z", "completedAt": "2026-11-04T23:47:58.781Z", "ms": 835584, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "50de0120-f0c0-4a5a-8383-7683969274cd" ] }, { "ref": "fieldnote-studio-office/7", "stepId": "draft-invoice", "activatedAt": "2026-11-04T23:34:03.255Z", "completedAt": "2026-11-04T23:56:34.432Z", "ms": 1351177, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "74f6d168-c8f4-4cea-b8e9-dc0f34ecd34e" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-04T23:03:30.823Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-04T23:03:30.713Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 447455 }, "sessions": { "total": 10, "taskScoped": 10, "byGezel": { "chiamaka": 1, "isadora": 6, "nils": 2, "pallavi": 1 }, "perTask": { "fieldnote-studio-office/1": 4, "fieldnote-studio-office/3": 1, "fieldnote-studio-office/6": 1, "fieldnote-studio-office/7": 1, "fieldnote-studio-office/4": 1, "fieldnote-studio-office/2": 1, "fieldnote-studio-office/5": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.8333333333333334, "maxMessages": 8, "maxContextFill": 0.2092437744140625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2092437744140625 }, "fanout": { "hosts": 1, "childrenSpawned": 6, "childrenCompleted": 6, "childrenFailed": 0, "barrierHolds": 2, "barrierReleases": 2, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 3 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-invoice-run", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "scenarioId": "craftbook-invoice-run", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/workspace/fieldnote-studio-office/report.md", "artifactBytes": 787, "artifactSha256": "a84ca7d325df51e97cff8ec11b8fa46375e076bb86ffd677be12dae935d56c00", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/in/invoice-run/versions/1.1.9/test.json", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ], "scoreAxes": { "numeric integrity": 8, "numbering continuity": 8, "professionalism": 6, "completeness": 4, "print-polish": 8 }, "meanScore": 6.8, "justification": "All three amounts and their 5415 total match uninvoiced ledger entries; invoice numbers are consecutive and terms-based due dates agree. Print layouts are clear with explicit margins. However, books still show the old last number and uninvoiced work; invoices omit issue date and currency, while the report adds an unconfirmed dollar sign.", "reviewBasis": "Read final report, all invoices, ledger/roster, supporting artifacts, task state and recorded tool events. Earlier screenshots match final 042/044 hashes; final 043 was separately rendered and visually reviewed after its total-row CSS changed. Print-CSS previews do not assert PDF pagination. Tested the actual read-only verdict validator against submitted and persisted text.", "deliverables": [ { "number": "2026-042", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/workspace/fieldnote-studio-office/invoices/2026-042.html", "bytes": 4433, "sha256": "02d686cf5c98a3fc7e7108723a96c074b906b0fccebd55cd48be02a6b681993a", "preview": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/advisory-invoice-preview/2026-042.png", "layout": { "scrollWidth": 635, "viewportWidth": 635, "scrollHeight": 846, "invoice": { "x": 0, "y": 0, "width": 635, "height": 390, "top": 0, "right": 635, "bottom": 390, "left": 0 }, "pageRules": [ "@page { size: letter; margin: 28mm 24mm; }" ] } }, { "number": "2026-043", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/workspace/fieldnote-studio-office/invoices/2026-043.html", "bytes": 4471, "sha256": "692727cf7bae2983c5213a0fb1474e3194245bae182db8a2a0c61514ae1a2ed8", "preview": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/advisory-invoice-final-preview/2026-043.png", "layout": { "scrollWidth": 635, "viewportWidth": 635, "scrollHeight": 846, "invoice": { "x": 0, "y": 0, "width": 635, "height": 372, "top": 0, "right": 635, "bottom": 372, "left": 0 }, "pageRules": [ "@page { size: letter; margin: 28mm 24mm; }" ] } }, { "number": "2026-044", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/workspace/fieldnote-studio-office/invoices/2026-044.html", "bytes": 4402, "sha256": "32633418ac05f23e5a1ac5d73a1bab6e93c4130a5f3ee73ca14def23aecf0641", "preview": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/advisory-invoice-preview/2026-044.png", "layout": { "scrollWidth": 635, "viewportWidth": 635, "scrollHeight": 846, "invoice": { "x": 0, "y": 0, "width": 635, "height": 390, "top": 0, "right": 635, "bottom": 390, "left": 0 }, "pageRules": [ "@page { size: letter; margin: 28mm 24mm; }" ] } } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:34.371Z", "reviewId": "037", "cellId": "r1-saluki-cuda-complex-work-craftbook-invoice-run", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "scenarioId": "craftbook-invoice-run", "status": "scored", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-invoice-run", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "fb9eb2190a767c36cdb9003efb1ddb8586d083187ef18494c7ab34ded8f86cf2", "evidenceSha256": "840fbc475b32b048f223365f7114fa9a3c599a948b87b564f9ca93d009d09702", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/workspace/fieldnote-studio-office/report.md", "bytes": 787, "sha256": "a84ca7d325df51e97cff8ec11b8fa46375e076bb86ffd677be12dae935d56c00", "shownBytes": 787, "truncated": false } ], "contextFiles": [], "scoreAxes": { "numeric integrity": 5, "numbering continuity": 6, "professionalism": 4, "completeness": 5, "print-polish": 2 }, "justification": "Only report.md is shown. It lists 042–044 continuing from 041, a consistent $5,415 total (1,840+2,600+975), and Ondaatje skipped on purpose with a reason. No invoice HTML, ledger or books update is visible, so amounts, due-date terms and the CSS can't be checked.", "limitations": "No invoice files, ledger, scope.md or billables.json were provided. Amounts, terms and numbering are checked only against the summary's own claims. Print styling (@page, margins, fonts) is unobservable. The books update is unverified.", "meanScore": 4.4, "durationMs": 16588, "costUsd": 0.022062, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3817, "cache_read_input_tokens": 0, "output_tokens": 679, "output_tokens_details": { "thinking_tokens": 335 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3817, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 679, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 3817, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3817 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 679, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 3817, "webSearchRequests": 0, "costUSD": 0.022062, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 335, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3817, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-invoice-run / underdog-saluki-27b-iq2-mix\n\n**Composite: 5.5 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 58m 44s; budget used: 56.5%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\\\" stuck for 15m — 3 re-writes (28 tool calls) without sniff movement. Artifact produced b...`; `facts.timing.timeToFirstArtifactMs: 363109`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":783,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...` |\n| Output quality (25%) | 9 / 10 | The latest scenario gate reached 90% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 363109`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":783,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.5645034448648641`; `facts.toolUse.totalToolCalls: 86`; `facts.timing.timeToFirstArtifactMs: 363109`; `facts.outcome.durationMs: 3524475` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 9 + 0.2 * 2.5 + 0.15 * 10 = **5.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14701 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30866 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.44 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 294488 + 21159 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 12 → 9 | `continuity.steps` |\n| Gate rejections / redrives | 1 / 0 | `continuity.steps` |\n| Median step wall-clock | 447s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 10 / 0.83 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 21% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 6 / 6 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 2 / 2 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 3 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 7 of 9 / 35 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\\\" stuck for 15m — 3 re-writes (28 tool calls) without sniff movement. Artifact produced b...`\n- `facts.timing.timeToFirstArtifactMs: 363109`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":783,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 90% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 363109`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":783,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.5645034448648641`\n- `facts.toolUse.totalToolCalls: 86`\n- `facts.timing.timeToFirstArtifactMs: 363109`\n- `facts.outcome.durationMs: 3524475`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\\\" stuck for 15m — 3 re-writes (28 tool calls) without sniff movement. Artifact produced b...`; `facts.timing.timeToFirstArtifactMs: 363109`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":783,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...`\n- Quality: The latest scenario gate reached 90% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 363109`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":783,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.5645034448648641`; `facts.toolUse.totalToolCalls: 86`; `facts.timing.timeToFirstArtifactMs: 363109`; `facts.outcome.durationMs: 3524475`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result: **fail, 5.5/10**. Separate codex-inline/current-session advisory mean: **6.8/10**.\n\nRead final report, all invoices, ledger/roster, supporting artifacts, task state and recorded tool events. Earlier screenshots match final 042/044 hashes; final 043 was separately rendered and visually reviewed after its total-row CSS changed. Print-CSS previews do not assert PDF pagination. Tested the actual read-only verdict validator against submitted and persisted text.\n\n| Declared axis | Score |\n|---|---:|\n| numeric integrity | 8/10 |\n| numbering continuity | 8/10 |\n| professionalism | 6/10 |\n| completeness | 4/10 |\n| print-polish | 8/10 |\n\nAll three amounts and their 5415 total match uninvoiced ledger entries; invoice numbers are consecutive and terms-based due dates agree. Print layouts are clear with explicit margins. However, books still show the old last number and uninvoiced work; invoices omit issue date and currency, while the report adds an unconfirmed dollar sign.\n\n## Positives\n\n- The primary report and three invoices correctly bill 1840, 2600 and 975, total 5415, exclude the previously billed 1200, and intentionally skip the idle roster client (sniff.latest.score: 9/10; final source-to-output review). No extra invented invoice or amount was found.\n- Invoice numbers 2026-042 through 2026-044 continue 2026-041 without gaps or repeats. Due dates 2026-12-04, 2026-11-18 and 2026-12-04 match the explicit November 4 scope date and net 30/14/30 terms (sniff.latest signals; source roster, scope and final invoices).\n- All three final invoices have readable line items and totals, Letter @page margins of 28mm/24mm, print CSS and clean visual hierarchy. The final Kestrel revision puts the total in the table and fits a 372px content height; the unchanged other invoices fit 390px. No horizontal overflow appears in the reviewed print-CSS previews (artifacts.htmlFiles; advisory render audits).\n\n## Negatives\n\n- The task stops active in collect pass 2, after the standard runner reports a 15-minute retry loop with three report rewrites and 28 calls without gate movement. It never reaches finish (outcome.success: false; failureMode: model-stuck; sniff.latest.score: 9/10; captured task state). This is a workflow failure despite usable invoices.\n- The ledger remains at lastInvoiceNumber 2026-041 and all three newly billed entries remain uninvoiced. The declared completeness axis asks for updated books, but the installed craftbook has no explicit ledger-update step or corresponding review criterion (sniff.latest gate omits this check; final ledger and craftbook state). Re-running this fixture workflow could bill the same entries again.\n- At 23:34:03 the saved verdict is rejected as structurally invalid. The seven submitted criterion lines all end PASS and pass the actual lineReviewVerdictFailure function; the runtime-appended unindented Sources heading/entries make the persisted file fail. This triggers a second three-child invoice fanout (toolUse.byTool.write_artifact: 4; task gateAttemptHistory; submitted/persisted validator probe). The reviewer did not submit the extra section.\n- Sniff feedback then resumes Nils session 9cd37441, bound to the old collect activation at 23:09:28.410Z, against the new activation at 23:34:03.330Z. Explicit advances at 23:43:39 and 23:56:47 are rejected stale_task_step and observable auto-advance also refuses. The resumed session continues reading and rewriting instead of ending; the classifier records three read-tool-repeat aborts (toolUse.byTool.read_file: 39; advance_task_step: 9; daemon/recording).\n- Numeric grounding repeatedly refuses correctly derived/formatted amounts, reporting fragments 840.00 and 415.00 despite 1840 and 5415 evidence. The writer responds with citation/rewrite cycles (toolUse.byTool.write_file: 14; failed tool events). This is a concrete formatting/evidence-handling symptom, not proof that every refused claim is valid.\n- Invoices omit an explicit issue date and currency, while the report assumes dollars without a currency in the seed data. The report says invoices went out, but the evidence shows local fixture files only, with no external sending action (sniff.latest output evidence; final artifacts and toolUse.byTool).\n\n## Tactical experiment — not applied\n\nIn ../gilde/data/craftbook-templates/in/invoice-run/versions/1.1.9/craftbook.json, add an explicit idempotent ledger-commit step after review approval: preserve each source entry identity, store its approved invoice number, mark only included entries invoiced and advance the last number once. Add a corresponding bookkeeping criterion and require issue date/currency from explicit business settings, asking when currency is absent. This is a craftbook contract experiment, not model sampling advice. Measure duplicate billing prevention on two consecutive runs and validate an unrelated ledger/reconciliation workflow before claiming general improvement.\n\n## Strategic experiments — not applied\n\n1. Reconcile packages/service/src/chat/evidence-ledger.ts source-footnote augmentation with packages/service/src/tasks/review-verdict.ts. Keep authoritative citations as structured metadata or explicitly parse a bounded Sources footer outside criterion lines, while still rejecting ambiguous/failed criteria. The exact submitted text passes and the automatically augmented text fails the current function. Add regression coverage for a valid cited verdict and an invalid criterion; measure needless reviewer rejection across unrelated code/document craftbooks.\n2. In evals/src/task-step-routing.ts and evals/src/sniff-feedback.ts, require current activation identity when choosing a repair session. A rejected step reactivated with the same id must receive a fresh owning session; never resume a predecessor. Keep the existing HTTP stale-step guard. Measure stale_task_step repairs, repeated reads and terminal completion on invoice and live-edit workflows.\n3. In packages/service/src/chat/evidence-ledger.ts and its numeric claim extraction dependency, test grouped decimal amounts against canonical numeric evidence; avoid interpreting the tail of 1,840.00 as 840.00. Validate derived totals with explicit operands rather than silent assertion acceptance. Compare invoice, API review and A/B readout grounding false positives.\n\n## Performance and fit\n\nElapsed 58.74 minutes; 21159 generated tokens; mean decode 8.44 tokens/sec and prefill 381.48 tokens/sec. Peak VRAM 10644/12227 MiB; process RSS 14701 MiB; whole-system RAM 30866/31865 MiB (perf fields). One native 0.1.57 CUDA launch, 65536-token context, f16 KV, one slot, 4096 reasoning budget. Dense FFN 39/64 layers on GPU, 25 on CPU: mixed residency. Shared GPU memory and paging are not separately measured. No explicit OOM/context overflow/endpoint failure/unexpected native exit found. The recorded termination is a retry-loop outcome, not a load failure.\n\n## Generalization and next experiment\n\nThe unrelated code-review and API review trials also show evidence-grounding friction with valid numbers/line references; edit-midtask shows ownership/version conflicts; bug-fix-tdd completed with real execution receipts. Test citation-safe verdict persistence and activation-aware repair routing across those different task types. No prompt, tuning, gate, source or weight was changed in this campaign; no failed capability trial was rerun.\n\n## Last three same-scenario trials\n\n- [craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/facts.json): pass, fixed 8.5, 192.4 seconds, 142.73 recorded tokens/sec. craftbook-invoice-run passed 10 deterministic craftbook checks\n- [craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/facts.json): pass, fixed 9, 197.1 seconds, 137.89 recorded tokens/sec. craftbook-invoice-run passed 10 deterministic craftbook checks\n- [craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T16-58-17-389Z-0ql1/facts.json): pass, fixed 9, 179.5 seconds, 139.54 recorded tokens/sec. craftbook-invoice-run passed 10 deterministic craftbook checks\n\nEarlier outcomes: 3/3 passed. Model/native/source identities differ, so this is historical context rather than a controlled native-version comparison.\n\n## Strategic note\n\nThe observed verdict failure is reproducible from framework-produced bytes: a valid submitted review becomes invalid after automatic source attribution. The subsequent stale-session repair compounds the failure. Correct invoice amounts do not establish workflow completion or bookkeeping safety. Preserve both symptoms separately from native GPU fit and fixed scores.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-workflow-audit.json, historical-comparison.json, the preserved provisional checkpoint, and both invoice-preview folders.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-author-linear", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-linear", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T00:05:17.290Z", "finishedAt": "2026-11-05T00:47:02.643Z", "durationMs": 2505353, "success": true, "reason": "authored a gated 3-step craftbook, invoked it, ran the task to completion, and all three outputs pass their checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 686040 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-linear", "bytes": 7783, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.25079627860751896 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 47 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 686040 }, { "source": "facts", "path": "outcome.durationMs", "value": 2505353 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 420, "peakRssMb": 15260, "peakCpuPercent": 1237.2, "firstAtMs": 0, "lastAtMs": 2488546 }, "gpu": { "available": true, "sampleCount": 420, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 420, "peakUsedMb": 31269, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 122858, "totalOutputTokens": 16081, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 126372, "todayTokensCached": 118060, "todayTokensOut": 1080, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 126372, "totalTokensCached": 118060, "totalTokensOut": 1080, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 12403, "cachedInputTokens": 12077, "outputTokens": 75, "cost": 0, "durationMs": 932611, "outputTokensPerSec": 8.44918131429299, "at": "2026-11-05T00:46:51.090Z" }, "lastUpdated": "2026-11-05T00:46:51.090Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T00:46:53.469Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2489920, "meanTokensPerSec": 8.31, "genTokensPerSec": 8.31, "genTokens": 16081, "promptTokensPerSec": 329.65, "peakRssMb": 15260, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 126372, "output": 1080 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "order-intake-cleanup/1", "stepId": "inspect", "activatedAt": "2026-11-05T00:12:40.795Z", "completedAt": "2026-11-05T00:18:51.135Z", "ms": 370340, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "1738b1e8-8b11-48f7-b391-b9182c55b12c" ] }, { "ref": "order-intake-cleanup/1", "stepId": "clean", "activatedAt": "2026-11-05T00:18:51.137Z", "completedAt": "2026-11-05T00:30:15.095Z", "ms": 683958, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "1738b1e8-8b11-48f7-b391-b9182c55b12c" ] }, { "ref": "order-intake-cleanup/1", "stepId": "verify", "activatedAt": "2026-11-05T00:30:15.097Z", "completedAt": "2026-11-05T00:46:51.013Z", "ms": 995916, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "26549cfd-21d1-42e2-b122-224d9063b7b6" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T00:05:23.077Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T00:05:22.970Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 683958 }, "sessions": { "total": 4, "taskScoped": 2, "byGezel": { "fatoumata": 1, "frederik": 2, "reza": 1 }, "perTask": { "order-intake-cleanup/1": 2 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.4, "maxMessages": 9, "maxContextFill": 0.3624114990234375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3624114990234375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/reorder-run", "knowledge://bendyline/handboek/whats-new/1.26273", "knowledge://bendyline/handboek/craftbook/scrape-to-structured" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "authored a gated 3-step craftbook, invoked it, ran the task to completion, and all three outputs pass their checks", "durationMs": 2505353, "timeoutMs": 9989594, "budgetUsedFraction": 0.25079627860751896 }, "timing": { "startedAt": "2026-11-05T00:05:17.290Z", "finishedAt": "2026-11-05T00:47:02.643Z", "timeToFirstArtifactMs": 686040, "timeToLastArtifactWriteMs": 2493801, "timeToFirstTokenMs": 47651, "firstTurnTtftMs": 37176, "timeToFirstToolCallMs": 58604 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 47, "byTool": { "list_craftbooks": 1, "list_dir": 4, "craftbook_read": 2, "craftbook_write": 1, "invoke_craftbook": 1, "get_task": 5, "read_task_notes": 6, "read_file": 9, "advance_task_step": 4, "stat": 5, "message_gezel": 1, "list_tasks": 1, "write_file": 3, "write_task_note": 3, "validate": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 7695, "filePath": "craftbook-author-linear", "bytes": 0, "score": 3, "scoreMax": 7, "signals": [], "failReason": "no reusable authored craftbook with at least 3 steps exists yet — author \"CSV Order Cleanup\" (inspect, clean, verify) as a craftbook template, not as ad-hoc chat work" }, { "atMs": 414898, "filePath": "craftbook-author-linear", "bytes": 28, "score": 3, "scoreMax": 7, "signals": [], "failReason": "no task has been created from craftbook \"csv-order-cleanup\" yet — invoke the craftbook on this project" }, { "atMs": 444375, "filePath": "craftbook-author-linear", "bytes": 28, "score": 3, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 686858, "filePath": "craftbook-author-linear", "bytes": 1657, "score": 4, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 1315570, "filePath": "craftbook-author-linear", "bytes": 5462, "score": 5, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 2327706, "filePath": "craftbook-author-linear", "bytes": 7783, "score": 6, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 2495713, "filePath": "craftbook-author-linear", "bytes": 7783, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-author-linear", "bytes": 7783, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 2495706, "kind": "structured", "gezel": "reza", "question": "**CSV Order Cleanup — run on data/orders.csv** is finished. - **`out/report.md`…", "chose": "write-in (workspace-fixture)" } ] }, "miscEvents": [ "[2026-11-05T00:05:17.293Z] [trial] id=craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz scenario=craftbook-author-linear model=underdog-saluki-27b-iq2-mix", "[2026-11-05T00:05:17.293Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "[2026-11-05T00:05:17.293Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T00:05:17.293Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-linear-ybD6C3", "[2026-11-05T00:05:17.293Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T00:05:17.296Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T00:05:17.299Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T00:05:17.305Z] [trial] throughput-scaled ceiling: 80m → 166m at 9.61 tok/s (reference 20 tok/s)", "[2026-11-05T00:05:17.305Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-linear-ybD6C3", "[2026-11-05T00:05:23.045Z] [trial] daemon spawned pid=3932 port=https://127.0.0.1:54801", "[2026-11-05T00:05:24.422Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T00:05:24.436Z] [trial] running scenario setup hook", "[2026-11-05T00:05:24.436Z] [trial] meester=fenton", "[2026-11-05T00:05:24.750Z] [trial] scenario setup complete", "[2026-11-05T00:05:24.752Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T00:05:24.820Z] [poll] initial digests hard=f868f174396dc68a soft=39849290e446a140", "[2026-11-05T00:05:25.120Z] [poll] hard-progress: f868f174396dc68a → 507301505cb40134 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=3/0B)", "[2026-11-05T00:06:17.720Z] [poll] hard-progress: 507301505cb40134 → 429944eea0c09262 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=39,stream=64) sniff=3/0B)", "[2026-11-05T00:06:35.411Z] [poll] hard-progress: 429944eea0c09262 → 4248d77953ab2850 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=60,stream=118) sniff=3/0B)", "[2026-11-05T00:06:47.010Z] [poll] hard-progress: 4248d77953ab2850 → 2f37b0f87de00713 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=79,stream=173) sniff=3/0B)", "[2026-11-05T00:12:12.289Z] [poll] hard-progress: 2f37b0f87de00713 → 8d0a7d500ea2b49a after 325s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=922,stream=3798) sniff=3/28B)", "[2026-11-05T00:12:41.752Z] [poll] hard-progress: 8d0a7d500ea2b49a → 6b7525459a184a5c after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=979,stream=4047) sniff=3/28B)", "[2026-11-05T00:12:53.391Z] [poll] hard-progress: 6b7525459a184a5c → 8a26d186ca72fffc after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=999,stream=4108) sniff=3/28B)", "[2026-11-05T00:13:22.861Z] [poll] hard-progress: 8a26d186ca72fffc → fc1500a3dc70a1e4 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=1037,stream=4214) sniff=3/28B)", "[2026-11-05T00:13:46.682Z] [poll] hard-progress: fc1500a3dc70a1e4 → 9a48e3affa5363d5 after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=1098,stream=4395) sniff=3/28B)" ] }, "metrics": { "process": { "sampleCount": 420, "peakRssMb": 15260, "peakCpuPercent": 1237.2, "firstAtMs": 0, "lastAtMs": 2488546 }, "gpu": { "available": true, "sampleCount": 420, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 420, "peakUsedMb": 31269, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 122858, "totalOutputTokens": 16081, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 126372, "todayTokensCached": 118060, "todayTokensOut": 1080, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 126372, "totalTokensCached": 118060, "totalTokensOut": 1080, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 12403, "cachedInputTokens": 12077, "outputTokens": 75, "cost": 0, "durationMs": 932611, "outputTokensPerSec": 8.44918131429299, "at": "2026-11-05T00:46:51.090Z" }, "lastUpdated": "2026-11-05T00:46:51.090Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T00:46:53.469Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2489920, "meanTokensPerSec": 8.31, "genTokensPerSec": 8.31, "genTokens": 16081, "promptTokensPerSec": 329.65, "peakRssMb": 15260, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 126372, "output": 1080 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T00:05:17.290Z", "finishedAt": "2026-11-05T00:47:02.643Z", "timeToFirstArtifactMs": 686040, "timeToLastArtifactWriteMs": 2493801, "timeToFirstTokenMs": 47651, "firstTurnTtftMs": 37176, "timeToFirstToolCallMs": 58604 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "order-intake-cleanup/1", "stepId": "inspect", "activatedAt": "2026-11-05T00:12:40.795Z", "completedAt": "2026-11-05T00:18:51.135Z", "ms": 370340, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "1738b1e8-8b11-48f7-b391-b9182c55b12c" ] }, { "ref": "order-intake-cleanup/1", "stepId": "clean", "activatedAt": "2026-11-05T00:18:51.137Z", "completedAt": "2026-11-05T00:30:15.095Z", "ms": 683958, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "1738b1e8-8b11-48f7-b391-b9182c55b12c" ] }, { "ref": "order-intake-cleanup/1", "stepId": "verify", "activatedAt": "2026-11-05T00:30:15.097Z", "completedAt": "2026-11-05T00:46:51.013Z", "ms": 995916, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "26549cfd-21d1-42e2-b122-224d9063b7b6" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T00:05:23.077Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T00:05:22.970Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 683958 }, "sessions": { "total": 4, "taskScoped": 2, "byGezel": { "fatoumata": 1, "frederik": 2, "reza": 1 }, "perTask": { "order-intake-cleanup/1": 2 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.4, "maxMessages": 9, "maxContextFill": 0.3624114990234375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3624114990234375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-author-linear", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "scenarioId": "craftbook-author-linear", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/workspace/order-intake-cleanup/out/report.md", "artifactBytes": 2335, "artifactSha256": "cb924436ba6edca4e9ec7900c10eae464174b7fa5057abb8d0787dd460a7e985", "axisSource": "evals/src/craftbooks/authoring/author-linear.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. All 28 valid records are retained faithfully, exactly two malformed rows dropped, and the report reconciles counts and catches the anomaly log’s wrong customer name. The authored three-step task completes. Reusability remains limited by an unused input parameter and size/nonempty-only recipe gates.", "reviewBasis": "Read final three outputs and verify hashes against the prior full source-row audit; reviewed final task, validation receipts, failed format call and terminal advance." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:40.910Z", "reviewId": "038", "cellId": "r1-saluki-cuda-complex-work-craftbook-author-linear", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "scenarioId": "craftbook-author-linear", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-linear", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "1aa35e2f89f66396fdedcdca2f45086e7449609e59c28c726a6e115d650f86c4", "evidenceSha256": "ca0ea522dd0cc01b205872a9a37d98639db2992807def8330feebd37a213014e", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/workspace/order-intake-cleanup/out/report.md", "bytes": 2335, "sha256": "cb924436ba6edca4e9ec7900c10eae464174b7fa5057abb8d0787dd460a7e985", "shownBytes": 2335, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz\\workspace\\order-intake-cleanup\\data\\orders.csv", "bytes": 1546, "sha256": "c88785ed78c4d8b14484d8d0f04516c9c93c99045b2e90b60b19f96ee17d7bb2", "shownBytes": 1546, "truncated": false } ], "scoreAxes": {}, "justification": "No numeric axes declared. The shown report.md is consistent with the CSV: 30 rows, ORD-1013 (qty \"two\") and ORD-1022 (date 2026-13-45) dropped, 28 kept (≥20). It also flags a customer-name error in the anomaly log. The report is coherent, but authoring and invocation are unshown.", "limitations": "Only report.md and the input CSV were shown. The craftbook document, craftbook_write and invoke_craftbook calls, step deliverable declarations, orders.json and anomalies.md were not inspected. Authoring order and task completion are unverified.", "meanScore": null, "durationMs": 14212, "costUsd": 0.02002, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4189, "cache_read_input_tokens": 700, "output_tokens": 319, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4189, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 319, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 4189, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4189 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 319, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 4189, "webSearchRequests": 0, "costUSD": 0.02002, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4189, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-linear / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 41m 45s; budget used: 25.1%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 686040`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7783,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.25079627860751896`; `facts.toolUse.totalToolCalls: 47`; `facts.timing.timeToFirstArtifactMs: 686040`; `facts.outcome.durationMs: 2505353` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15260 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31269 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.31 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 122858 + 16081 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 684s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 2 / 0.40 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 1 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 36% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 686040`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7783,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.25079627860751896`\n- `facts.toolUse.totalToolCalls: 47`\n- `facts.timing.timeToFirstArtifactMs: 686040`\n- `facts.outcome.durationMs: 2505353`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 686040`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7783,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.25079627860751896`; `facts.toolUse.totalToolCalls: 47`; `facts.timing.timeToFirstArtifactMs: 686040`; `facts.outcome.durationMs: 2505353`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result: **pass, 8.5/10**. No qualitative axes are declared for this scenario; the codex-inline/current-session review is prose-only.\n\nRead final three outputs and verify hashes against the prior full source-row audit; reviewed final task, validation receipts, failed format call and terminal advance.\n\nNo declared axes. All 28 valid records are retained faithfully, exactly two malformed rows dropped, and the report reconciles counts and catches the anomaly log’s wrong customer name. The authored three-step task completes. Reusability remains limited by an unused input parameter and size/nonempty-only recipe gates.\n\n## Positives\n\n- The model authored one reusable local CSV Order Cleanup recipe, invoked it once, and completed inspect, clean and verify with each file gate satisfied (outcome.success; sniff.latest.score: 7/7; toolUse.byTool.craftbook_write: 1; invoke_craftbook: 1; captured terminal task).\n- Every field of all 28 retained records matches its raw CSV row after equivalent numeric formatting is normalized. Only ORD-1013 and ORD-1022 are dropped; no duplicate or invented records were found. All three final files match the hashes from the prior full review (sniff.latest.bytes: 7783 characters across outputs/step ids; advisory record audit).\n- The 2335-byte findings report accurately reconciles the counts, explains dropping malformed rows, and catches the anomaly log’s incorrect customer for ORD-1013. A real validate call confirms JSON parseability and 3805-byte nonempty data; the actual verify advance completes at 00:46:51.089Z (toolUse.byTool.validate: 1; write_file: 3; advance_task_step: 4).\n\n## Negatives\n\n- The anomaly log misidentifies ORD-1013 as Linden Florists instead of Pixel & Frame. The report flags this, but the log remains uncorrected. Its claim that spelled-out quantity two was not repairable per the notes is stronger than those notes; dropping was nevertheless allowed (sniff.latest full gate; final anomaly/report content).\n- The saved clean gate checks only minimum size and nonempty content; verify likewise checks only report size/nonempty, rather than JSON schema, row preservation or a valid verdict. Actual records are correct, but the authored recipe’s gates alone do not establish semantic cleaning on another dataset (sniff.latest 7/7; final task.craftbook).\n- The declared exportPath parameter is not interpolated into the recipe prompts, which still name data/orders.csv. Only the default invocation was exercised, so a different input path is unproven (toolUse.byTool.craftbook_write: 1; saved paramSchema and craftbookParams).\n- An initial craftbook_read call uses invalid format full-document. The tool accepts summary/json/markdown; the harness AUTHORING_TOOL_STEER and AUTHORING_WORKER_ABOUT explicitly say full-document format, supplying a concrete ambiguity. The model recovers with a second read (toolUse.byTool.craftbook_read: 2; failed call at 00:06:34.084Z; helpers.ts lines 55/77).\n- Forty-seven calls, 16081 generated tokens and 41m45s are substantial overhead for this dataset and recipe. Five stat, five get_task and six read_task_notes calls add monitoring work alongside execution (toolUse.totalToolCalls/byTool; perf.derived.genTokens; outcome.durationMs).\n\n## Tactical experiment — not applied\n\nIn AUTHORING_TOOL_STEER and AUTHORING_WORKER_ABOUT in evals/src/craftbooks/authoring/helpers.ts, replace “full-document format” with “the complete document using one of the format values advertised by craftbook_read; do not pass full-document as an enum value.” This corrects a tool-contract ambiguity without choosing a codec or adding fixture-specific knowledge. Measure invalid-format calls on author-linear and export/generalize tasks; preserve format A/B neutrality.\n\n## Strategic experiment — not applied\n\nIn the reusable craftbook validation path in packages/mcp/src/server.ts, add an advisory unused-parameter check: declared input paths should appear as interpolations or an explicit binding in the relevant steps. For structured outputs, recommend a parse/schema gate rather than nonempty-only. Validate parameter use with a second, differently named CSV fixture and an unrelated supplier/report recipe; require source-record preservation and a single terminal run. Do not silently rewrite authored recipes or loosen gates. This is a future experiment; no campaign source was changed.\n\n## Performance and fit\n\nElapsed 41.76 minutes; 16081 recorded generated tokens; mean decode 8.31 tokens/sec, prefill 329.65 tokens/sec. Peak VRAM 10644/12227 MiB; process RSS 15260 MiB; whole-system RAM 31269/31865 MiB (perf fields). One native 0.1.57 CUDA launch; 65536-token context, f16 KV, one slot, 4096 reasoning budget. Dense FFN from 39/64 layers on GPU and 25 on CPU, as explicitly logged. This is mixed residency, not complete GPU residency; shared GPU memory/paging are not separately measured. No explicit OOM, context overflow, endpoint failure or unexpected native exit found; the final SIGTERM is expected trial shutdown.\n\n## Generalization and next experiment\n\nThis completed workflow supplies evidence beyond merely loading or authoring: real data survives source comparison and the task reaches terminal status. Unlike invoice-run, it has no gate rejection or obsolete-session repair. Its unused parameter and weak semantic gates still limit a claim of general reuse. Compare future authoring-contract changes against the unrelated route-multi and TDD flows, tracking invalid tool args, output correctness and completion, not only gate scores.\n\n## Last three same-scenario trials\n\n- [craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/facts.json): fail, fixed 5.4, 1039.6 seconds, 112.19 recorded tokens/sec. chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 004b00e149342d8c) (re-engage nudge was sent and ignored)\n- [craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s/facts.json): pass, fixed 8.5, 389.4 seconds, 113.38 recorded tokens/sec. authored a gated 3-step craftbook, invoked it, ran the task to completion, and all three outputs pass their checks\n- [craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-07-51-852Z-isgb/facts.json): fail, fixed 4.6, 203.9 seconds, 104.24 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: CSV Order Cleanup; missing signals stayed unchanged: task order-intake-cleanup/2 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count., out/report.md is 0 bytes, need ≥ 80, out/orders.json does not exist yet. Last failure: task order-intake-cleanup/2 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with\n\nEarlier outcomes: 1/3 passed. Different model/native/source identities make these historical context rather than a controlled native-version comparison. No tuning, prompt, gate, source or weight changed; no capability failure retried.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-workflow-audit.json and historical-comparison.json.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-route-multi", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-route-multi", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T00:50:32.682Z", "finishedAt": "2026-11-05T01:22:51.145Z", "durationMs": 1938463, "success": true, "reason": "invoked the bundled press-release recipe, ran the task to completion, and the release carries a dateline, the attributed approved quote, the boilerplate, a contact, and the approved facts rather than the stale draft", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 649193 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-route-multi", "bytes": 1064, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.18375013069400484 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 35 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 649193 }, { "source": "facts", "path": "outcome.durationMs", "value": 1938463 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 326, "peakRssMb": 15217, "peakCpuPercent": 1248.7, "firstAtMs": 0, "lastAtMs": 1928234 }, "gpu": { "available": true, "sampleCount": 326, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 326, "peakUsedMb": 31245, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 108197, "totalOutputTokens": 12006, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 141827, "todayTokensCached": 129442, "todayTokensOut": 5643, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 141827, "totalTokensCached": 129442, "totalTokensOut": 5643, "totalCost": 0, "medianOutputTokensPerSec": 8.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.2, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7646, "cachedInputTokens": 7285, "outputTokens": 416, "cost": 0, "durationMs": 157969, "outputTokensPerSec": 8.128433957306527, "at": "2026-11-05T01:22:45.669Z" }, "lastUpdated": "2026-11-05T01:22:45.669Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T01:22:47.183Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1929289, "meanTokensPerSec": 8.21, "genTokensPerSec": 8.21, "genTokens": 12006, "promptTokensPerSec": 341.98, "peakRssMb": 15217, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 141827, "output": 5643 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "fieldstone-launch-comms/1", "stepId": "scope", "activatedAt": "2026-11-05T00:51:53.872Z", "completedAt": "2026-11-05T01:01:22.070Z", "ms": 568198, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "a287f2ee-4f50-4b32-8c80-9fa2c4c78f6c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "draft", "activatedAt": "2026-11-05T01:01:22.072Z", "completedAt": "2026-11-05T01:08:36.851Z", "ms": 434779, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "a6c65bb1-cfcd-4109-9c24-e41759afe33e" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "review", "activatedAt": "2026-11-05T01:08:36.852Z", "completedAt": "2026-11-05T01:16:27.612Z", "ms": 470760, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "fc47151a-da3b-4841-b1e4-a764ba28968c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "evaluate", "activatedAt": "2026-11-05T01:16:27.613Z", "completedAt": "2026-11-05T01:20:04.286Z", "ms": 216673, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "fc47151a-da3b-4841-b1e4-a764ba28968c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "finish", "activatedAt": "2026-11-05T01:20:04.287Z", "completedAt": "2026-11-05T01:22:45.606Z", "ms": 161319, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "45fc93f6-c9c8-4268-a7bf-9e3deee79b87" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T00:50:37.976Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T00:50:37.891Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 434779 }, "sessions": { "total": 5, "taskScoped": 4, "byGezel": { "gudny": 1, "magdolna": 1, "oleksandr": 1, "shunsuke": 1, "zara": 1 }, "perTask": { "fieldstone-launch-comms/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 9, "maxContextFill": 0.41607666015625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.41607666015625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/gezel-in-docker", "knowledge://bendyline/handboek/notifications", "knowledge://bendyline/handboek/adding-folders", "knowledge://bendyline/handboek/craftbook/chapter-weave" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "invoked the bundled press-release recipe, ran the task to completion, and the release carries a dateline, the attributed approved quote, the boilerplate, a contact, and the approved facts rather than the stale draft", "durationMs": 1938463, "timeoutMs": 10549451, "budgetUsedFraction": 0.18375013069400484 }, "timing": { "startedAt": "2026-11-05T00:50:32.682Z", "finishedAt": "2026-11-05T01:22:51.145Z", "timeToFirstArtifactMs": 649193, "timeToLastArtifactWriteMs": 1771432, "timeToFirstTokenMs": 44881, "firstTurnTtftMs": 27000, "timeToFirstToolCallMs": 81336 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 35, "byTool": { "start_work": 1, "read_task_notes": 7, "get_task": 3, "write_task_note": 2, "read_artifact": 2, "list_artifacts": 3, "read_file": 8, "write_artifact": 2, "write_file": 2, "advance_task_step": 3, "list_dir": 1, "validate": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 6674, "filePath": "craftbook-route-multi", "bytes": 0, "score": 9, "scoreMax": 11, "signals": [], "failReason": "no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" }, { "atMs": 84026, "filePath": "craftbook-route-multi", "bytes": 0, "score": 9, "scoreMax": 11, "signals": [], "failReason": "task fieldstone-launch-comms/1 (from \"press-release\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 841847, "filePath": "craftbook-route-multi", "bytes": 1064, "score": 10, "scoreMax": 11, "signals": [], "failReason": "task fieldstone-launch-comms/1 (from \"press-release\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 1934274, "filePath": "craftbook-route-multi", "bytes": 1064, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-route-multi", "bytes": 1064, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 1934275, "kind": "structured", "gezel": "zara", "question": "Your document is finished. - **`press-release.md`** - `tasks/1/verdict.md` - `o…", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-05T00:50:32.686Z] [trial] id=craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp scenario=craftbook-route-multi model=underdog-saluki-27b-iq2-mix", "[2026-11-05T00:50:32.686Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-route-multi-b9K2D7", "[2026-11-05T00:50:32.686Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "[2026-11-05T00:50:32.687Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T00:50:32.686Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T00:50:32.690Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T00:50:32.694Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T00:50:32.698Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-route-multi-b9K2D7", "[2026-11-05T00:50:32.698Z] [trial] throughput-scaled ceiling: 80m → 176m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T00:50:37.869Z] [trial] daemon spawned pid=51308 port=https://127.0.0.1:55061", "[2026-11-05T00:50:38.774Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T00:50:38.791Z] [trial] running scenario setup hook", "[2026-11-05T00:50:38.791Z] [trial] meester=zara", "[2026-11-05T00:50:39.104Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T00:50:39.104Z] [trial] scenario setup complete", "[2026-11-05T00:50:39.173Z] [poll] initial digests hard=fbb2f2f240a0d026 soft=0df3030ef851dcbf", "[2026-11-05T00:50:39.440Z] [poll] hard-progress: fbb2f2f240a0d026 → f6e5a81e697189ef after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-05T00:51:56.766Z] [poll] hard-progress: f6e5a81e697189ef → 34c323a8c5937207 after 77s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=108,stream=382) sniff=9/0B)", "[2026-11-05T00:52:14.514Z] [poll] hard-progress: 34c323a8c5937207 → 135c08c2cafd7f6a after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=145,stream=496) sniff=9/0B)", "[2026-11-05T00:52:20.464Z] [poll] hard-progress: 135c08c2cafd7f6a → 535d1e44d960db80 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=152,stream=519) sniff=9/0B)", "[2026-11-05T00:53:01.813Z] [poll] hard-progress: 535d1e44d960db80 → 5e8bd94f4ac1fb0b after 41s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=3,slot=208,stream=673) sniff=9/0B)", "[2026-11-05T00:55:40.995Z] [poll] hard-progress: 5e8bd94f4ac1fb0b → 7b62a1cccccb4e23 after 159s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=629,stream=2043) sniff=9/0B)", "[2026-11-05T00:56:16.968Z] [poll] hard-progress: 7b62a1cccccb4e23 → dcef5e5d15377949 after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=646,stream=2084) sniff=9/0B)", "[2026-11-05T00:56:58.303Z] [poll] hard-progress: dcef5e5d15377949 → 2439310158aaa8f9 after 41s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=739,stream=2364) sniff=9/0B)", "[2026-11-05T00:57:21.956Z] [poll] hard-progress: 2439310158aaa8f9 → 08a0872771b88ff2 after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=789,stream=2512) sniff=9/0B)" ] }, "metrics": { "process": { "sampleCount": 326, "peakRssMb": 15217, "peakCpuPercent": 1248.7, "firstAtMs": 0, "lastAtMs": 1928234 }, "gpu": { "available": true, "sampleCount": 326, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 326, "peakUsedMb": 31245, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 108197, "totalOutputTokens": 12006, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 141827, "todayTokensCached": 129442, "todayTokensOut": 5643, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 141827, "totalTokensCached": 129442, "totalTokensOut": 5643, "totalCost": 0, "medianOutputTokensPerSec": 8.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.2, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7646, "cachedInputTokens": 7285, "outputTokens": 416, "cost": 0, "durationMs": 157969, "outputTokensPerSec": 8.128433957306527, "at": "2026-11-05T01:22:45.669Z" }, "lastUpdated": "2026-11-05T01:22:45.669Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T01:22:47.183Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1929289, "meanTokensPerSec": 8.21, "genTokensPerSec": 8.21, "genTokens": 12006, "promptTokensPerSec": 341.98, "peakRssMb": 15217, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 141827, "output": 5643 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T00:50:32.682Z", "finishedAt": "2026-11-05T01:22:51.145Z", "timeToFirstArtifactMs": 649193, "timeToLastArtifactWriteMs": 1771432, "timeToFirstTokenMs": 44881, "firstTurnTtftMs": 27000, "timeToFirstToolCallMs": 81336 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "fieldstone-launch-comms/1", "stepId": "scope", "activatedAt": "2026-11-05T00:51:53.872Z", "completedAt": "2026-11-05T01:01:22.070Z", "ms": 568198, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "a287f2ee-4f50-4b32-8c80-9fa2c4c78f6c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "draft", "activatedAt": "2026-11-05T01:01:22.072Z", "completedAt": "2026-11-05T01:08:36.851Z", "ms": 434779, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "a6c65bb1-cfcd-4109-9c24-e41759afe33e" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "review", "activatedAt": "2026-11-05T01:08:36.852Z", "completedAt": "2026-11-05T01:16:27.612Z", "ms": 470760, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "fc47151a-da3b-4841-b1e4-a764ba28968c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "evaluate", "activatedAt": "2026-11-05T01:16:27.613Z", "completedAt": "2026-11-05T01:20:04.286Z", "ms": 216673, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "fc47151a-da3b-4841-b1e4-a764ba28968c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "finish", "activatedAt": "2026-11-05T01:20:04.287Z", "completedAt": "2026-11-05T01:22:45.606Z", "ms": 161319, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "45fc93f6-c9c8-4268-a7bf-9e3deee79b87" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T00:50:37.976Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T00:50:37.891Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 434779 }, "sessions": { "total": 5, "taskScoped": 4, "byGezel": { "gudny": 1, "magdolna": 1, "oleksandr": 1, "shunsuke": 1, "zara": 1 }, "perTask": { "fieldstone-launch-comms/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 9, "maxContextFill": 0.41607666015625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.41607666015625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-route-multi", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "scenarioId": "craftbook-route-multi", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/workspace/fieldstone-launch-comms/out/press-release.md", "artifactBytes": 1066, "artifactSha256": "1e5839e497f08ea3e74c1468f10d816760d846585b6dd8fa49116f4e40fe1c77", "axisSource": "evals/src/craftbooks/authoring/route-multi.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The concise release preserves the approved announcement, figures, quote wording, title and contact while excluding stale facts. The bundled recipe completes. The requested output and default-path duplicate are identical; task parameters failed to bind the requested path, and the reviewer’s validate call checks only nonempty content.", "reviewBasis": "Read approved facts, stale teaser, scope, both complete releases, reviewer verdict, final task parameters and tool chronology; verified both delivered copies have identical hashes." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:45.605Z", "reviewId": "039", "cellId": "r1-saluki-cuda-complex-work-craftbook-route-multi", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "scenarioId": "craftbook-route-multi", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-route-multi", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "5a0c25d112f8d64f32e1973f2a6b7d9b87a52617f698de4cf31ca366549a8e17", "evidenceSha256": "5e42a8d798687f2c2b589ae2a4853f2b11202bd08e609ca30b4afa349c4a2a8f", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/workspace/fieldstone-launch-comms/out/press-release.md", "bytes": 1066, "sha256": "1e5839e497f08ea3e74c1468f10d816760d846585b6dd8fa49116f4e40fe1c77", "shownBytes": 1066, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The release is present, press-formatted (headline, dateline, quote, boilerplate, contact, ###), with a date and quote consistent with the brief. One awkward redundancy: \"founded in 2019 by Tenhoven\" repeats the earlier founding line. No fact errors can be checked.", "limitations": "The facts file, old teaser and tool-call trace are not shown. I can't verify the numbers, the exact quote, or the boilerplate against the source. I also can't confirm suggest_craftbook was used, the recipe was followed, or start_project was avoided. Source-only review.", "meanScore": null, "durationMs": 14512, "costUsd": 0.011494, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2065, "cache_read_input_tokens": 700, "output_tokens": 316, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2065, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 316, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2065, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2065 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 316, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2065, "webSearchRequests": 0, "costUSD": 0.011494, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2065, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-route-multi / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 32m 18s; budget used: 18.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 649193`; `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":1064,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.18375013069400484`; `facts.toolUse.totalToolCalls: 35`; `facts.timing.timeToFirstArtifactMs: 649193`; `facts.outcome.durationMs: 1938463` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15217 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31245 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.21 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 108197 + 12006 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 435s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 42% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 649193`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":1064,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.18375013069400484`\n- `facts.toolUse.totalToolCalls: 35`\n- `facts.timing.timeToFirstArtifactMs: 649193`\n- `facts.outcome.durationMs: 1938463`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 649193`; `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":1064,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.18375013069400484`; `facts.toolUse.totalToolCalls: 35`; `facts.timing.timeToFirstArtifactMs: 649193`; `facts.outcome.durationMs: 1938463`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result: **pass, 8.5/10**. No qualitative axes are declared for this scenario; the codex-inline/current-session review is prose-only.\n\nRead approved facts, stale teaser, scope, both complete releases, reviewer verdict, final task parameters and tool chronology; verified both delivered copies have identical hashes.\n\nNo declared axes. The concise release preserves the approved announcement, figures, quote wording, title and contact while excluding stale facts. The bundled recipe completes. The requested output and default-path duplicate are identical; task parameters failed to bind the requested path, and the reviewer’s validate call checks only nonempty content.\n\n## Positives\n\n- The model starts the bundled press-release 1.0.6 workflow and completes all five steps; it authors no competing recipe. Both route and output checks pass (outcome.success; sniff.latest.score: 11/11; toolUse.byTool.start_work: 1; final task.sourceCraftbookIds/status). This measures execution as well as selection.\n- The 1066-byte release includes a specific headline, dateline, clear lede, attributed approved quote wording, boilerplate, complete contact block and closing mark. Opening date 14 October 2026 and managing-director title correctly replace the abandoned teaser’s September date and CEO title. The 68 percent/40 km/11 staff/twice-monthly figures match the approved source (sniff.latest output checks; final artifact and facts/announcement.md).\n- The requested out/press-release.md is written at 01:04:34.471Z and survives unchanged through completion. The reviewer reads the requested copy and source facts; the default copy is byte-identical, so review did not approve different prose from the delivered file (toolUse.byTool.write_file: 2; read_file: 8; hash and event audit).\n\n## Negatives\n\n- The user destination is present in start_work details and a task note, but task.craftbookParams.outputPath remains press-release.md. The writer produces both the requested file and a default-path duplicate to satisfy the recipe. This creates two output owners/locations and consumes an extra full-file write (toolUse.byTool.write_file: 2; captured invocation and final parameters).\n- Zara tries read_artifact for the workspace release and receives a nonretryable authority error. The reviewer later reads it through the correct workspace tool; this is recovered surface/role friction, not missing content (failed call at 01:09:43.535Z; toolUse.byTool.read_artifact: 2; final artifact).\n- The validate receipt only checks file-non-empty, not truth, quotation fidelity or document structure. The reviewer’s separate manual checks and the scenario gate provide that evidence. The approved quote’s final period is changed to a comma for attribution, so quote wording is preserved but the review’s literal verbatim claim is imprecise (toolUse.byTool.validate: 1; final verdict/source comparison).\n- Thirty-five calls and 12006 generated tokens take 32m18s. Seven task-note reads, three artifact listings and three get_task calls add coordination overhead around a short document. The scope also marks the destination checklist done before either file is written (toolUse.byTool; outcome.durationMs; scope/write chronology).\n\n## Tactical experiment — not applied\n\nIn the scope prompt of ../gilde/data/craftbook-templates/pr/press-release/versions/1.0.6/craftbook.json, ask for unchecked acceptance criteria until a current artifact/receipt supports them. Keep the source/destination names as parameters rather than fixed examples. Measure premature completion claims and repeated polling on press-release and unrelated invoice/report workflows; do not change factual acceptance gates.\n\n## Strategic experiment — not applied\n\nIn packages/mcp/src/craftbook-routing.ts inferCraftbookJobParams and the start_work path in packages/mcp/src/server.ts, bind an explicitly requested output destination to the existing outputPath parameter before task creation. The recipe already declares outputPath; passing the destination only in prose left the default in final task state. Validate the inference with prose and binary-document workflows, assert one canonical deliverable path, and preserve authority checks. Separately prefer artifact descriptors that expose workspace/drawer location to readers, avoiding read_artifact calls against workspace files. These are future experiments, not changes to this run.\n\n## Performance and fit\n\nElapsed 32.31 minutes; 12006 recorded generated tokens; mean decode 8.21 tokens/sec, prefill 341.98 tokens/sec. Peak VRAM 10646/12227 MiB; process RSS 15217 MiB; whole-system RAM 31245/31865 MiB (perf fields). One native 0.1.57 CUDA launch; 65536-token context, f16 KV, one slot, 4096 reasoning budget. Dense FFN from 39/64 layers on GPU and 25 on CPU, as explicitly logged. This is mixed residency, not complete GPU residency; shared GPU memory/paging are not separately measured. No explicit OOM, context overflow, endpoint failure or unexpected native exit found; the final SIGTERM is expected trial shutdown.\n\n## Generalization and next experiment\n\nSelection-only dev-craftbook-routing and find-vs-create do not demonstrate writing; this trial does, with source-faithful final prose. Compare the destination-binding issue against duplicate or stale artifacts in edit-midtask and document workflows. The invoice trial’s automatically appended Sources footer failure did not occur here: this verdict has no footer and the installed press-release gate uses a different pattern. Do not treat this pass as disproving the separately reproduced invoice framework defect.\n\n## Last three same-scenario trials\n\n- [craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/facts.json): fail, fixed 5.5, 303.2 seconds, 99.49 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/press-release.md; missing signals stayed unchanged: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project. Last failure: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project\n- [craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl/facts.json): pass, fixed 8.5, 629.7 seconds, 114.39 recorded tokens/sec. invoked the bundled press-release recipe, ran the task to completion, and the release carries a dateline, the attributed approved quote, the boilerplate, a contact, and the approved facts rather than the stale draft\n- [craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-35-11-678Z-4hbj/facts.json): fail, fixed 5.5, 343.3 seconds, 109.25 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on out/press-release.md; missing signals stayed unchanged: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project. Last failure: no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project\n\nEarlier outcomes: 1/3 passed. Different model/native/source identities make these historical context rather than a controlled native-version comparison. No tuning, prompt, gate, source or weight changed; no capability failure retried.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-workflow-audit.json and historical-comparison.json.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-export-generalize", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-export-generalize", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T01:25:52.409Z", "finishedAt": "2026-11-05T02:35:29.901Z", "durationMs": 4177492, "success": false, "reason": "repair-aborted: reza/b9450c59 exhausted its bounded automatic recovery allowance at the current checked progress; last error: The model got stuck repeating the same action without making progress, so the turn was stopped. Try sending your message again, or rephrase your request.", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-export-generalize", "score": 1, "bytes": 26, "deliverableMissing": true, "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-aborted: reza/b9450c59 exhausted its bounded automatic recovery allowance at the current checked progress; last error: The model got stuck repeating the same action without making progress, so the turn was stopped. Try sending your message again, or rephrase your request." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 3474511 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 26, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" } } ] }, "quality": { "score": 1.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 16.7% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 3474511 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 26, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.39599141225453344 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 34 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 3474511 }, { "source": "facts", "path": "outcome.durationMs", "value": 4177492 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 2.4, "band": "framework-gap" }, "facts": { "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 699, "peakRssMb": 14561, "peakCpuPercent": 1221, "firstAtMs": 0, "lastAtMs": 4164173 }, "gpu": { "available": true, "sampleCount": 699, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 699, "peakUsedMb": 30837, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 89544, "totalOutputTokens": 31871, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 43952, "todayTokensCached": 42611, "todayTokensOut": 1240, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 43952, "totalTokensCached": 42611, "totalTokensOut": 1240, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 16614, "cachedInputTokens": 15635, "outputTokens": 88, "cost": 0, "durationMs": 417539, "outputTokensPerSec": 8.41445545416298, "at": "2026-11-05T02:28:48.843Z" }, "lastUpdated": "2026-11-05T02:28:48.843Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T02:35:24.425Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4165858, "meanTokensPerSec": 8.32, "genTokensPerSec": 8.32, "genTokens": 31871, "promptTokensPerSec": 384.02, "peakRssMb": 14561, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43952, "output": 1240 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T01:25:58.064Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T01:25:57.963Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 3, "taskScoped": 0, "byGezel": { "linnea": 1, "reza": 1, "seongjun": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.19171142578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.19171142578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 3 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "repair-aborted: reza/b9450c59 exhausted its bounded automatic recovery allowance at the current checked progress; last error: The model got stuck repeating the same action without making progress, so the turn was stopped. Try sending your message again, or rephrase your request.", "durationMs": 4177492, "timeoutMs": 10549451, "budgetUsedFraction": 0.39599141225453344 }, "timing": { "startedAt": "2026-11-05T01:25:52.409Z", "finishedAt": "2026-11-05T02:35:29.901Z", "timeToFirstArtifactMs": 3474511, "timeToLastArtifactWriteMs": 3776435, "timeToFirstTokenMs": 47589, "firstTurnTtftMs": 37398, "timeToFirstToolCallMs": 56587 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Data Clerk (Supplier File Processing)", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 34, "byTool": { "read_file": 7, "list_dir": 1, "stat": 1, "list_craftbooks": 1, "craftbook_read": 2, "ensure_gezel": 1, "craftbook_write": 2, "save_memory": 9, "search": 1, "list_project_gezels": 1, "delegate_builder": 1, "read_artifact": 2, "search_memory": 3, "list_artifacts": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 7639, "filePath": "craftbook-export-generalize", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 4016415, "filePath": "craftbook-export-generalize", "bytes": 26, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" } ], "latest": { "filePath": "craftbook-export-generalize", "bytes": 26, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T01:25:52.412Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "[2026-11-05T01:25:52.412Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T01:25:52.412Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T01:25:52.412Z] [trial] id=craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus scenario=craftbook-export-generalize model=underdog-saluki-27b-iq2-mix", "[2026-11-05T01:25:52.412Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-export-generalize-766bDU", "[2026-11-05T01:25:52.415Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T01:25:52.419Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T01:25:52.424Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-export-generalize-766bDU", "[2026-11-05T01:25:52.424Z] [trial] throughput-scaled ceiling: 80m → 176m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T01:25:58.027Z] [trial] daemon spawned pid=28676 port=https://127.0.0.1:60654", "[2026-11-05T01:25:59.452Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T01:25:59.467Z] [trial] meester=linnea", "[2026-11-05T01:25:59.467Z] [trial] running scenario setup hook", "[2026-11-05T01:25:59.787Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T01:25:59.785Z] [trial] scenario setup complete", "[2026-11-05T01:25:59.859Z] [poll] initial digests hard=366988e71b162170 soft=c2093faa6b9956b1", "[2026-11-05T01:26:00.149Z] [poll] hard-progress: 366988e71b162170 → ca0ba0a3aa8c0c3e after 0s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T01:26:52.984Z] [poll] hard-progress: ca0ba0a3aa8c0c3e → 6ec20eba4a4630c0 after 53s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=36,stream=66) sniff=1/0B)", "[2026-11-05T01:27:10.722Z] [poll] hard-progress: 6ec20eba4a4630c0 → 510bda33d4cc6222 after 18s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=69,stream=168) sniff=1/0B)", "[2026-11-05T01:27:16.561Z] [poll] hard-progress: 510bda33d4cc6222 → 69e5969fc9f4a8b1 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=73,stream=176) sniff=1/0B)", "[2026-11-05T01:35:53.435Z] [poll] hard-progress: 69e5969fc9f4a8b1 → 3a0c3468ddefbdb6 after 517s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=1451,stream=4275) sniff=1/0B)", "[2026-11-05T01:39:51.488Z] [poll] hard-progress: 3a0c3468ddefbdb6 → 329248a8670ed806 after 238s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=1996,stream=5887) sniff=1/0B)", "[2026-11-05T01:52:07.274Z] [poll] hard-progress: 329248a8670ed806 → 0cf895b4ce978a4b after 736s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=10,slot=3902,stream=12919) sniff=1/0B)", "[2026-11-05T02:00:46.683Z] [poll] hard-progress: 0cf895b4ce978a4b → c0e3e44f8703ffe7 after 519s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=11,slot=5323,stream=17309) sniff=1/0B)", "[2026-11-05T02:01:34.179Z] [poll] poisoned-session recovery: reza/b9450c59 last turn aborted; sending one direct repair turn" ] }, "metrics": { "process": { "sampleCount": 699, "peakRssMb": 14561, "peakCpuPercent": 1221, "firstAtMs": 0, "lastAtMs": 4164173 }, "gpu": { "available": true, "sampleCount": 699, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 699, "peakUsedMb": 30837, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 89544, "totalOutputTokens": 31871, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 43952, "todayTokensCached": 42611, "todayTokensOut": 1240, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 43952, "totalTokensCached": 42611, "totalTokensOut": 1240, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 16614, "cachedInputTokens": 15635, "outputTokens": 88, "cost": 0, "durationMs": 417539, "outputTokensPerSec": 8.41445545416298, "at": "2026-11-05T02:28:48.843Z" }, "lastUpdated": "2026-11-05T02:28:48.843Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T02:35:24.425Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4165858, "meanTokensPerSec": 8.32, "genTokensPerSec": 8.32, "genTokens": 31871, "promptTokensPerSec": 384.02, "peakRssMb": 14561, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43952, "output": 1240 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T01:25:52.409Z", "finishedAt": "2026-11-05T02:35:29.901Z", "timeToFirstArtifactMs": 3474511, "timeToLastArtifactWriteMs": 3776435, "timeToFirstTokenMs": 47589, "firstTurnTtftMs": 37398, "timeToFirstToolCallMs": 56587 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T01:25:58.064Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T01:25:57.963Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 3, "taskScoped": 0, "byGezel": { "linnea": 1, "reza": 1, "seongjun": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.19171142578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.19171142578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 3 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-export-generalize", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "scenarioId": "craftbook-export-generalize", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/workspace/supplier-intake/out/q2-intake.md", "artifactBytes": 0, "axisSource": "evals/src/craftbooks/authoring/export-generalize.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "None of the four requested Q1/Q2 outputs exists. A separate inspection report and an uninvoked template were produced, but they do not complete either quarter. Nine memory writes culminate in a repetition abort. Recovery also removes authoring tools, memory search rejects structured provenance, and the saved recipe has an invalid regex.", "reviewBasis": "Checked all four declared paths in the final capture; read the separate report, both source datasets, saved craftbook tool payload, failures, recovery log and final state. Reviewed auxiliary outputs in prose without awarding scores for absent requested artifacts.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/workspace/supplier-intake/out/q1-intake.md", "status": "noArtifact" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/workspace/supplier-intake/out/q1-clean.json", "status": "noArtifact" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/workspace/supplier-intake/out/q2-intake.md", "status": "noArtifact" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/workspace/supplier-intake/out/q2-clean.json", "status": "noArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:49.097Z", "reviewId": "040", "cellId": "r1-saluki-cuda-complex-work-craftbook-export-generalize", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "scenarioId": "craftbook-export-generalize", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-export-generalize", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "3b667cb13b9ab21119aa90bde6dcf07ab0c716815f30ca55f455b3ee6b64853d", "evidenceSha256": "5cec180dcfcfd72f01eccbf29d7cfe82c18070f58ee2c814912e9bbd19375d51", "artifactFiles": [], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus\\workspace\\supplier-intake\\data\\suppliers-q1.json", "bytes": 1260, "sha256": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "shownBytes": 1260, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus\\workspace\\supplier-intake\\data\\suppliers-q2.json", "bytes": 1263, "sha256": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "shownBytes": 1263, "truncated": false } ], "scoreAxes": {}, "justification": "The declared deliverable workspace/supplier-intake/out/q2-intake.md was missing at closeout. Only the Q1 and Q2 input files were provided; no outputs, craftbook, or tool-call logs. Inputs show impossible lead times (Q1: 0, -5; Q2: 0, -14), but nothing shows the recipe was saved or invoked.", "limitations": "No transcript, craftbook, Q1 outputs, or run logs were shown, so I can't tell whether the run stopped early, failed, or wrote files elsewhere. This is not a model failure or success verdict, and no scores are given.", "meanScore": null, "durationMs": 14683, "costUsd": 0.019484, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4050, "cache_read_input_tokens": 700, "output_tokens": 321, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4050, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 321, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 4050, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4050 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 321, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 4050, "webSearchRequests": 0, "costUSD": 0.019484, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4050, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-export-generalize / underdog-saluki-27b-iq2-mix\n\n**Composite: 2.4 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 69m 37s; budget used: 39.6%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-aborted: reza/b9450c59 exhausted its bounded automatic recovery allowance at the current checked progress; last error: The model got stuck repeating the same action with...`; `facts.timing.timeToFirstArtifactMs: 3474511`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":26,\"score\":1,\"scoreMax\":6,\"signals\":[],\"failReason\":\"out/q1-intake.md does not exist yet — write the short Q1 intake report the...` |\n| Output quality (25%) | 1.7 / 10 | The latest scenario gate reached 16.7% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 3474511`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":26,\"score\":1,\"scoreMax\":6,\"signals\":[],\"failReason\":\"out/q1-intake.md does not exist yet — write the short Q1 intake report the...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.39599141225453344`; `facts.toolUse.totalToolCalls: 34`; `facts.timing.timeToFirstArtifactMs: 3474511`; `facts.outcome.durationMs: 4177492` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 1.7 + 0.2 * 2.5 + 0.15 * 10 = **2.4**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14561 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10632 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30837 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.32 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 89544 + 31871 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 19% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 3 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"repair-aborted: reza/b9450c59 exhausted its bounded automatic recovery allowance at the current checked progress; last error: The model got stuck repeating the same action with...`\n- `facts.timing.timeToFirstArtifactMs: 3474511`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":26,\"score\":1,\"scoreMax\":6,\"signals\":[],\"failReason\":\"out/q1-intake.md does not exist yet — write the short Q1 intake report the...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 16.7% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 3474511`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":26,\"score\":1,\"scoreMax\":6,\"signals\":[],\"failReason\":\"out/q1-intake.md does not exist yet — write the short Q1 intake report the...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.39599141225453344`\n- `facts.toolUse.totalToolCalls: 34`\n- `facts.timing.timeToFirstArtifactMs: 3474511`\n- `facts.outcome.durationMs: 4177492`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-aborted: reza/b9450c59 exhausted its bounded automatic recovery allowance at the current checked progress; last error: The model got stuck repeating the same action with...`; `facts.timing.timeToFirstArtifactMs: 3474511`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":26,\"score\":1,\"scoreMax\":6,\"signals\":[],\"failReason\":\"out/q1-intake.md does not exist yet — write the short Q1 intake report the...`\n- Quality: The latest scenario gate reached 16.7% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 3474511`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":26,\"score\":1,\"scoreMax\":6,\"signals\":[],\"failReason\":\"out/q1-intake.md does not exist yet — write the short Q1 intake report the...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.39599141225453344`; `facts.toolUse.totalToolCalls: 34`; `facts.timing.timeToFirstArtifactMs: 3474511`; `facts.outcome.durationMs: 4177492`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result: **fail, 2.4/10**. All four declared outputs are **noArtifact**; no qualitative scores are invented. The scenario declares no judge axes. The auxiliary report and template receive prose review only.\n\nNone of the four requested Q1/Q2 outputs exists. A separate inspection report and an uninvoked template were produced, but they do not complete either quarter. Nine memory writes culminate in a repetition abort. Recovery also removes authoring tools, memory search rejects structured provenance, and the saved recipe has an invalid regex.\n\nChecked all four declared paths in the final capture; read the separate report, both source datasets, saved craftbook tool payload, failures, recovery log and final state. Reviewed auxiliary outputs in prose without awarding scores for absent requested artifacts.\n\n## Positives\n\n- Both raw files are read successfully and the auxiliary report correctly describes five fields, eight suppliers per quarter and the two negative lead-time entries. Reza’s memory notes correctly identify both zero and negative entries under the <=0 policy (toolUse.byTool.read_file: 7; write_file: 1; save_memory: 9; captured source/report/memory events). This is evidence of reading, not requested completion.\n- Linnea eventually saves a three-step parameterized template with quarter and inputPath interpolations, two gated output steps and a terminal finish step (toolUse.byTool.craftbook_write: 2, one successful; recorded tool result). The template is real, but it is never invoked or tested on Q2.\n\n## Negatives\n\n- All four requested report/clean-data paths are absent. The only generated workspace file is report.md, a combined inspection report; there is no cleaned Q1 or Q2 list and no task from the saved recipe. The sniff’s 26 bytes represent recipe step-id metadata, not a requested output (outcome.success: false; sniff.latest.score: 1/6; deliverableMissing true in campaign result; toolUse.byTool has no invoke/export).\n- The first authoring attempt fails because deliverable.kind markdown is invalid. The initial 86-tool authoring surface exposes craftbook_read/write and read_file but not write_file; the scenario asks for hand-produced Q1 workspace files before authoring while reusing a Crew Lead persona and anti-manual-authoring steering. A later amendment describes the exception but does not itself supply a workspace write tool (failed call 01:52:03.569Z; toolUse.byTool.craftbook_write: 2; helpers.ts/export-generalize.ts and roster audit).\n- After a no-output/context-gathering abort at 02:01:34, standard recovery activates project-orchestration and reduces Reza’s surface from 86 to 24 tools. The recovered surface retains save_memory and delegation but removes craftbook_read/write as well as direct file tools. The original request still expects that worker to author the recipe; this is a concrete recovery/tool-contract mismatch (outcome.failureMode: model-stuck; daemon roster transition; recorded error events).\n- Nine save_memory calls include repeated identical Q1 and Q2 entries. The terminal error at 02:35:19 reports repeated action with no progress; the standard runner exhausts its single automatic recovery and stops after 69m37s and 31871 generated tokens (toolUse.byTool.save_memory: 9; outcome.reason/durationMs; perf.derived.genTokens). No manual retry was performed.\n- The builder hits search_memory errors three times because matches[0].source is an object where SearchToolOutputSchema expects a string. server.ts passes memory results directly to that schema, while service memory results retain structured provenance. Changing the search query cannot fix this response validation failure (toolUse.byTool.search_memory: 3; failed events; packages/mcp/src/tool-contracts.ts and server.ts).\n- The auxiliary report treats zero-day lead time as only a warning, unlike Reza’s <=0 memory notes and the fixture intent. It accurately labels its different rule, so the handoff needed an explicit agreed policy rather than silent drift. Its claim currency is almost certainly EUR is unsupported, and its no-files-modified sentence ignores writing the report itself (sniff.latest failure; report content and source data).\n- The saved recipe instructs write_artifact while declaring workspace outputs/gates, retains invalid negative values in the cleaned set, and includes contains pattern (?i)impossible|lead time. JavaScript RegExp rejects that inline flag; core containsPattern constructs RegExp directly. This is an unexecuted latent recipe defect, not the observed cause of the earlier abort (toolUse.byTool.craftbook_write; saved payload; read-only regex probe).\n\n## Tactical experiment — not applied\n\nIn evals/src/craftbooks/authoring/helpers.ts, let the worker setup explicitly support a read-transform-write phase when the job requires concrete work before generalization; use that option from export-generalize.ts. Replace the contradictory author-first/never-hand-write steer with phase-scoped instructions: finish the current requested file receipts, then author and invoke the reusable procedure. Preserve the same final gates and separate manual first-pass work from recipe-produced second-pass work. Measure wrong-path work and memory-only loops on two-stage data and report workflows, not only this fixture.\n\n## Strategic experiments — not applied\n\n1. In packages/service/src/chat/session-tool-surface.ts projectOrchestrationConstraintActive, carry the original operation intent through generic recovery; an authoring repair must not lose craftbook_read/write, and an authorized workspace production phase must retain a route to writing. In evals/src/runner.ts recovery, assert the next required action remains available. Keep role permissions explicit rather than granting every worker all tools. Test authoring, direct-file and routing cases together.\n2. In packages/mcp/src/server.ts search_memory and tool-contracts.ts, map structured memory provenance into a dedicated typed field or schema union instead of passing an object to a string-only source field. Preserve provenance in the response and add an end-to-end recall fixture with source metadata; validate unrelated memory-assisted research as well as supplier processing.\n3. At craftbook_write validation in packages/mcp/src/server.ts, compile regex checks using the runtime’s own semantics before accepting a document, and flag artifact/workspace mismatches. The supplied (?i) form fails the actual JavaScript constructor; provide a precise correction to pattern plus flags rather than accepting a never-passing gate.\n\n## Performance and fit\n\nElapsed 69.62 minutes; 31871 recorded generated tokens; mean decode 8.32 tokens/sec, prefill 384.02 tokens/sec. Peak VRAM 10632/12227 MiB; process RSS 14561 MiB; whole-system RAM 30837/31865 MiB (perf fields). One native 0.1.57 CUDA launch, 65536-token context, f16 KV, one slot, 4096 reasoning budget. Dense FFN 39/64 layers on GPU and 25 on CPU, explicitly logged. Mixed residency; shared GPU memory/paging not separately measured. No explicit OOM/context overflow/endpoint failure/unexpected native exit found. The final SIGTERM is expected teardown after a workflow repetition failure.\n\n## Generalization and next experiment\n\nAuthor-linear demonstrates that this model/runtime can produce source-faithful structured output and complete a newly authored task; export-generalize fails the concrete-first handoff and subsequent recovery. Route-multi also exposed workspace/drawer read friction. First test intent-preserving recovery plus contract-correct memory results across authoring, document production and memory-assisted research. Measure retained required tools, successful file receipts, repeated writes and terminal completion. This does not justify a claim that loading implies diverse task competence, nor a recommendation to replace the model instead of investigating these concrete failure paths.\n\n## Last three same-scenario trials\n\n- [craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/facts.json): fail, fixed 3.3, 817.7 seconds, 103.51 recorded tokens/sec. chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 2d44e61df691a47b) (re-engage nudge was sent and ignored)\n- [craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n/facts.json): fail, fixed 2.4, 371.4 seconds, 102.98 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: supplier intake recipe; missing signals stayed unchanged: out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time, out/q1-clean.json does not exist yet — write the cleaned Q1 supplier records there as one list of supplier entries, task supplier-intake/1 (from craftbook \"supplier-intake-recipe\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a \n- [craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T17-56-34-227Z-fmhi/facts.json): fail, fixed 3.4, 761.8 seconds, 104.51 recorded tokens/sec. chat stalled for 303s — daemon reachable but issuing no model turns (soft digest stuck at 2da6f0fd3a4f605a) (re-engage nudge was sent and ignored)\n\nEarlier outcomes: 0/3 passed. Different model/native/source identities mean these are historical context, not a controlled native-version comparison.\n\n## Strategic note\n\nThe tool response schema rejects actual memory provenance, and recovery visibly removes authoring tools from the authoring worker. Preserve these framework findings alongside the model’s repeated memory calls and missing deliverables; the saved invalid regex remains an unexecuted risk. No runtime, gate, tuning, model or source changes were made and no capability failure was retried.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-workflow-audit.json and historical-comparison.json. Earlier noArtifact checkpoint remains preserved.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-author-params", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-params", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T02:38:35.029Z", "finishedAt": "2026-11-05T03:13:03.572Z", "durationMs": 2068543, "success": true, "reason": "authored one parameterized craftbook, ran it twice with different region inputs, and both region summaries exist with distinct contents", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1096808 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 286, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.17429389898932704 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 46 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1096808 }, { "source": "facts", "path": "outcome.durationMs", "value": 2068543 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 346, "peakRssMb": 16010, "peakCpuPercent": 1235.8, "firstAtMs": 0, "lastAtMs": 2053929 }, "gpu": { "available": true, "sampleCount": 346, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 346, "peakUsedMb": 31245, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 85231, "totalOutputTokens": 13815, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 93362, "todayTokensCached": 87240, "todayTokensOut": 1252, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 93362, "totalTokensCached": 87240, "totalTokensOut": 1252, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8431, "cachedInputTokens": 8254, "outputTokens": 65, "cost": 0, "durationMs": 277535, "outputTokensPerSec": 8.618126505478974, "at": "2026-11-05T03:11:18.055Z" }, "lastUpdated": "2026-11-05T03:11:18.055Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T03:12:56.894Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2055661, "meanTokensPerSec": 8.5, "genTokensPerSec": 8.5, "genTokens": 13815, "promptTokensPerSec": 335.62, "peakRssMb": 16010, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 93362, "output": 1252 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 4 }, "steps": { "activated": 6, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "regional-sales-rollup/1", "stepId": "compute", "activatedAt": "2026-11-05T02:52:46.645Z", "completedAt": "2026-11-05T02:59:32.945Z", "ms": 406300, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "20fbd383-20a3-492b-80c9-135a127d989a" ] }, { "ref": "regional-sales-rollup/1", "stepId": "summary", "activatedAt": "2026-11-05T02:59:32.946Z", "completedAt": "2026-11-05T03:11:17.969Z", "ms": 705023, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "20fbd383-20a3-492b-80c9-135a127d989a" ] }, { "ref": "regional-sales-rollup/2", "stepId": "compute", "activatedAt": "2026-11-05T02:52:47.072Z", "completedAt": "2026-11-05T03:06:37.149Z", "ms": 830077, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "8946bace-2ac5-4253-b383-bbafeb4e733a" ] }, { "ref": "regional-sales-rollup/2", "stepId": "summary", "activatedAt": "2026-11-05T03:06:37.150Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "8946bace-2ac5-4253-b383-bbafeb4e733a" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T02:38:40.780Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T02:38:40.684Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 705023 }, "sessions": { "total": 4, "taskScoped": 2, "byGezel": { "nada": 3, "reza": 1 }, "perTask": { "regional-sales-rollup/1": 1, "regional-sales-rollup/2": 1 }, "reusedAcrossSteps": 2, "continuityReuses": 2, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 5, "maxContextFill": 0.4701385498046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.4701385498046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 2, "tasks": 4, "tasksWithReferences": 2, "items": 10, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/month-close", "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/month-close", "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbook/scrape-to-structured" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "authored one parameterized craftbook, ran it twice with different region inputs, and both region summaries exist with distinct contents", "durationMs": 2068543, "timeoutMs": 11868132, "budgetUsedFraction": 0.17429389898932704 }, "timing": { "startedAt": "2026-11-05T02:38:35.029Z", "finishedAt": "2026-11-05T03:13:03.572Z", "timeToFirstArtifactMs": 1096808, "timeToLastArtifactWriteMs": 2056471, "timeToFirstTokenMs": 47590, "firstTurnTtftMs": 37170, "timeToFirstToolCallMs": 85100 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 46, "byTool": { "list_dir": 7, "list_craftbooks": 1, "read_file": 10, "craftbook_read": 1, "how_do_i": 2, "craftbook_write": 1, "invoke_craftbook": 2, "list_tasks": 8, "read_task_notes": 2, "get_task": 2, "write_file": 4, "write_task_note": 3, "advance_task_step": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 7675, "filePath": "craftbook-author-params", "bytes": 0, "score": 4, "scoreMax": 7, "signals": [], "failReason": "no reusable authored craftbook with at least 2 steps exists yet — author the regional sales rollup as a reusable craftbook template, not as ad-hoc chat work" }, { "atMs": 817336, "filePath": "craftbook-author-params", "bytes": 27, "score": 4, "scoreMax": 7, "signals": [], "failReason": "no task has been created from craftbook \"region-sales-rollup\" yet — start the recipe on this project for the north region" }, { "atMs": 852874, "filePath": "craftbook-author-params", "bytes": 27, "score": 5, "scoreMax": 7, "signals": [], "failReason": "out/north-summary.md is 0 bytes, need ≥ 80" }, { "atMs": 1858822, "filePath": "craftbook-author-params", "bytes": 159, "score": 6, "scoreMax": 7, "signals": [], "failReason": "out/south-summary.md is 0 bytes, need ≥ 80" }, { "atMs": 2061400, "filePath": "craftbook-author-params", "bytes": 286, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-author-params", "bytes": 286, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 1965649, "kind": "structured", "gezel": "reza", "question": "**Roll up north region sales** is finished. - **`out/north-summary.md`** - `out…", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-05T02:38:35.032Z] [trial] id=craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p scenario=craftbook-author-params model=underdog-saluki-27b-iq2-mix", "[2026-11-05T02:38:35.032Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "[2026-11-05T02:38:35.032Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-params-JbTBSs", "[2026-11-05T02:38:35.032Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T02:38:35.033Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T02:38:35.035Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T02:38:35.039Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T02:38:35.044Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-params-JbTBSs", "[2026-11-05T02:38:35.044Z] [trial] throughput-scaled ceiling: 90m → 198m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T02:38:40.740Z] [trial] daemon spawned pid=22776 port=https://127.0.0.1:63738", "[2026-11-05T02:38:42.124Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T02:38:42.141Z] [trial] meester=cassian", "[2026-11-05T02:38:42.141Z] [trial] running scenario setup hook", "[2026-11-05T02:38:42.487Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T02:38:42.485Z] [trial] scenario setup complete", "[2026-11-05T02:38:42.549Z] [poll] initial digests hard=5fcaf5af9c6ddec2 soft=c60a5fedb1c33dcc", "[2026-11-05T02:38:42.849Z] [poll] hard-progress: 5fcaf5af9c6ddec2 → 723eaebae21c3219 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=4/0B)", "[2026-11-05T02:40:05.053Z] [poll] hard-progress: 723eaebae21c3219 → 6689d2ea0d2fbf8e after 82s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=112,stream=286) sniff=4/0B)", "[2026-11-05T02:40:40.685Z] [poll] hard-progress: 6689d2ea0d2fbf8e → 4b80ba966ecdd430 after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=166,stream=433) sniff=4/0B)", "[2026-11-05T02:42:15.339Z] [poll] hard-progress: 4b80ba966ecdd430 → f61b0f43970eee80 after 95s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=402,stream=1151) sniff=4/0B)", "[2026-11-05T02:42:27.144Z] [poll] hard-progress: f61b0f43970eee80 → 51dacd3fa4618a2a after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=423,stream=1233) sniff=4/0B)", "[2026-11-05T02:52:12.490Z] [poll] hard-progress: 51dacd3fa4618a2a → a9d5734d7ee46a4f after 585s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=1996,stream=6703) sniff=4/27B)", "[2026-11-05T02:52:47.993Z] [poll] hard-progress: a9d5734d7ee46a4f → 9ee52c38e3cc3462 after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=2066,stream=7000) sniff=5/27B)", "[2026-11-05T02:52:59.903Z] [poll] hard-progress: 9ee52c38e3cc3462 → d094c2ff29214da2 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=2086,stream=7060) sniff=5/27B)", "[2026-11-05T02:53:11.716Z] [poll] hard-progress: d094c2ff29214da2 → 9bbe3fc16f5c45b7 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=13,slot=2105,stream=7123) sniff=5/27B)" ] }, "metrics": { "process": { "sampleCount": 346, "peakRssMb": 16010, "peakCpuPercent": 1235.8, "firstAtMs": 0, "lastAtMs": 2053929 }, "gpu": { "available": true, "sampleCount": 346, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 346, "peakUsedMb": 31245, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 85231, "totalOutputTokens": 13815, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 93362, "todayTokensCached": 87240, "todayTokensOut": 1252, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 93362, "totalTokensCached": 87240, "totalTokensOut": 1252, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8431, "cachedInputTokens": 8254, "outputTokens": 65, "cost": 0, "durationMs": 277535, "outputTokensPerSec": 8.618126505478974, "at": "2026-11-05T03:11:18.055Z" }, "lastUpdated": "2026-11-05T03:11:18.055Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T03:12:56.894Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2055661, "meanTokensPerSec": 8.5, "genTokensPerSec": 8.5, "genTokens": 13815, "promptTokensPerSec": 335.62, "peakRssMb": 16010, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 93362, "output": 1252 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T02:38:35.029Z", "finishedAt": "2026-11-05T03:13:03.572Z", "timeToFirstArtifactMs": 1096808, "timeToLastArtifactWriteMs": 2056471, "timeToFirstTokenMs": 47590, "firstTurnTtftMs": 37170, "timeToFirstToolCallMs": 85100 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 4 }, "steps": { "activated": 6, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "regional-sales-rollup/1", "stepId": "compute", "activatedAt": "2026-11-05T02:52:46.645Z", "completedAt": "2026-11-05T02:59:32.945Z", "ms": 406300, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "20fbd383-20a3-492b-80c9-135a127d989a" ] }, { "ref": "regional-sales-rollup/1", "stepId": "summary", "activatedAt": "2026-11-05T02:59:32.946Z", "completedAt": "2026-11-05T03:11:17.969Z", "ms": 705023, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "20fbd383-20a3-492b-80c9-135a127d989a" ] }, { "ref": "regional-sales-rollup/2", "stepId": "compute", "activatedAt": "2026-11-05T02:52:47.072Z", "completedAt": "2026-11-05T03:06:37.149Z", "ms": 830077, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "8946bace-2ac5-4253-b383-bbafeb4e733a" ] }, { "ref": "regional-sales-rollup/2", "stepId": "summary", "activatedAt": "2026-11-05T03:06:37.150Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "8946bace-2ac5-4253-b383-bbafeb4e733a" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T02:38:40.780Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T02:38:40.684Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 705023 }, "sessions": { "total": 4, "taskScoped": 2, "byGezel": { "nada": 3, "reza": 1 }, "perTask": { "regional-sales-rollup/1": 1, "regional-sales-rollup/2": 1 }, "reusedAcrossSteps": 2, "continuityReuses": 2, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 5, "maxContextFill": 0.4701385498046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.4701385498046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-author-params", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "scenarioId": "craftbook-author-params", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/workspace/regional-sales-rollup/out/north-summary.md", "artifactBytes": 132, "artifactSha256": "d688c6a4e997b50a885927a5e689efd18bc76856b17ec0e42a8b2878f97d22e3", "axisSource": "evals/src/craftbooks/authoring/author-params.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. One recipe produces correct distinct totals and top reps for both regions without being edited between invocations. North assumes dollars without source currency; formatting differs. North task completes, while south remains active when its summary triggers the scenario pass, so both files are verified but dual terminal completion is not.", "reviewBasis": "Read both final summaries, totals files, raw CSVs, saved recipe, invocation parameters, final tasks and tool chronology. Independently summed integer units and revenue cents and found each top representative; distinguished produced files from task lifecycle completion.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/workspace/regional-sales-rollup/out/north-summary.md", "bytes": 132, "sha256": "d688c6a4e997b50a885927a5e689efd18bc76856b17ec0e42a8b2878f97d22e3" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/workspace/regional-sales-rollup/out/south-summary.md", "bytes": 127, "sha256": "3190dec61d9ca19c7bdd7cb96d1a1444972ab3dfeaf0ba631a2f3c8c4de1f11e" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:56.732Z", "reviewId": "041", "cellId": "r1-saluki-cuda-complex-work-craftbook-author-params", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "scenarioId": "craftbook-author-params", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-params", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "a2c77ba6722eeb5de8daec4dac69a555ef42f8039f27c5b1012d65a42156e7fa", "evidenceSha256": "2d6899130f2e207fbc682ff9064909ee48bebbe01e5eb6db220d830a71db1559", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/workspace/regional-sales-rollup/out/north-summary.md", "bytes": 132, "sha256": "d688c6a4e997b50a885927a5e689efd18bc76856b17ec0e42a8b2878f97d22e3", "shownBytes": 132, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p\\workspace\\regional-sales-rollup\\data\\north.csv", "bytes": 331, "sha256": "1d565713b89e9dc055719ef769bf6ba5bea1ff888d309aa34767a5231a0af9ba", "shownBytes": 331, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p\\workspace\\regional-sales-rollup\\data\\south.csv", "bytes": 335, "sha256": "e68e801e3fc4700a2f4e7625ae8db017447668c1b36b0d19858bc309ac0be370", "shownBytes": 335, "truncated": false } ], "scoreAxes": {}, "justification": "No numeric axes are declared. The north summary is correct: units 1,298, revenue $47,847.95 and top rep Cato Meijer ($7,955.25) all match north.csv. No south-summary.md is shown, so the second run is unverified. The craftbook, its region parameter, and the tool-call order are also not shown.", "limitations": "Source-only review. The craftbook file, its parameterization, the invoke calls and the south output are not in the evidence, so reusability, the two-step structure, authoring before delegation, and the south run cannot be judged. Outcome is runner-terminal.", "meanScore": null, "durationMs": 15789, "costUsd": 0.01647, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2679, "cache_read_input_tokens": 700, "output_tokens": 568, "output_tokens_details": { "thinking_tokens": 257 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2679, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 568, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2679, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2679 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 568, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2679, "webSearchRequests": 0, "costUSD": 0.01647, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 257, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2679, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-params / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 34m 29s; budget used: 17.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1096808`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":286,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.17429389898932704`; `facts.toolUse.totalToolCalls: 46`; `facts.timing.timeToFirstArtifactMs: 1096808`; `facts.outcome.durationMs: 2068543` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 16010 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31245 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.5 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 85231 + 13815 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 4s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 6 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 705s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 2 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 2 (2 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 47% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 2 / 2 of 4 / 10 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1096808`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":286,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.17429389898932704`\n- `facts.toolUse.totalToolCalls: 46`\n- `facts.timing.timeToFirstArtifactMs: 1096808`\n- `facts.outcome.durationMs: 2068543`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1096808`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":286,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.17429389898932704`; `facts.toolUse.totalToolCalls: 46`; `facts.timing.timeToFirstArtifactMs: 1096808`; `facts.outcome.durationMs: 2068543`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result: **pass, 8.5/10**. This scenario declares no qualitative axes; codex-inline/current-session review is prose-only.\n\nRead both final summaries, totals files, raw CSVs, saved recipe, invocation parameters, final tasks and tool chronology. Independently summed integer units and revenue cents and found each top representative; distinguished produced files from task lifecycle completion.\n\nNo declared axes. One recipe produces correct distinct totals and top reps for both regions without being edited between invocations. North assumes dollars without source currency; formatting differs. North task completes, while south remains active when its summary triggers the scenario pass, so both files are verified but dual terminal completion is not.\n\n## Positives\n\n- One template is authored and invoked exactly twice, with region north and south respectively. Both tasks retain the same catalog id/version and resolved input/output paths for their region (toolUse.byTool.craftbook_write: 1; invoke_craftbook: 2; final task.craftbookParams/sourceCraftbookIds). This shows actual parameter use, unlike the unused exportPath in author-linear.\n- Independent source arithmetic confirms north 1298 units, 47847.95 revenue, top Cato Meijer at 7955.25; south 2270 units, 83990.00 revenue, top Nadia Roos at 15170.00. Both short summaries and intermediate totals files carry these correct figures (sniff.latest.score: 7/7; toolUse.byTool.write_file: 4; advisory source audit).\n- The second invocation is not implemented by editing the template or copying north’s values. The same developer reads each region CSV and writes separate totals and summary paths. North reaches terminal status at 03:11:18.053Z (toolUse.byTool.read_file: 10; advance_task_step: 3; recorded sequence).\n\n## Negatives\n\n- The scenario stops when south-summary.md is written at 03:12:51.504Z; south task remains active in summary, with no completion timestamp or final advance. The current successCheck requires two task invocations and distinct output files, not both tasks complete. Preserve the fixed pass, but do not report dual terminal completion (outcome.success; sniff.latest.score: 7/7; final task state; author-params.ts).\n- North prefixes revenue with a dollar sign despite no currency column or stated currency in the CSV. South leaves amounts unlabeled; heading case and numeric grouping also vary. The numerical values are correct, while the financial unit and presentation are inconsistent (sniff.latest output evidence; raw sources and final summaries).\n- The recipe gates only byte size and nonempty content; the scenario checks that summaries differ, not their arithmetic. Distinct wrong totals could satisfy those structural checks. This trial’s numerical correctness is established by the independent source audit (sniff.latest 7/7; final craftbook gates; author-params.ts successCheck).\n- Forty-six calls include eight list_tasks and seven list_dir calls while a single worker handles both tasks. An early read of south-summary.md returns not found before that write. The workflow succeeds but repeated polling contributes overhead to the 34m29s elapsed for two short summaries (toolUse.totalToolCalls/byTool; failed event at 03:11:32.542Z; outcome.durationMs).\n\n## Tactical experiment — not applied\n\nIn AUTHORING_WORKER_ABOUT in evals/src/craftbooks/authoring/helpers.ts, add a general data-report instruction: preserve source units and currency, label unknown units explicitly, and use the same declared presentation across parameterized runs. Do not invent a currency from location or names. Apply no fixture amounts or output strings. Measure unsupported units and cross-run consistency on regional rollups and unrelated invoice/budget reports.\n\n## Strategic experiments — not applied\n\nIn evals/src/craftbooks/authoring/author-params.ts, separately record both task terminal states and source-derived numerical agreement, rather than equating distinct nonempty summaries with completed reuse. If the intended contract includes terminal completion, wait for both existing tasks instead of stopping on the second file write. Preserve frozen scores in this campaign. In packages/service/src/chat/manager.ts, prefer task lifecycle notifications to repeated list_tasks/list_dir polling while the responsible worker is active; measure calls and latency on concurrent report and code tasks.\n\n## Performance and fit\n\nElapsed 34.48 minutes; 13815 recorded generated tokens; mean decode 8.5 tokens/sec, prefill 335.62 tokens/sec. Peak VRAM 10646/12227 MiB; process RSS 16010 MiB; whole-system RAM 31245/31865 MiB (perf fields). Process RSS is a new observed campaign high, distinct from whole-system RAM. One native 0.1.57 CUDA launch, 65536-token context, f16 KV, one slot and 4096 reasoning budget. Dense FFN from 39/64 layers on GPU, 25 on CPU. This is mixed residency, with shared GPU memory/paging not separately measured. No explicit OOM, context overflow, endpoint error or unexpected native exit; SIGTERM is expected trial teardown.\n\n## Generalization and next experiment\n\nThis trial demonstrates a parameter reaching two different source files and correctly changing both results. It is stronger reuse evidence than author-linear’s unused exportPath and export-generalize’s uninvoked template. Invoice-run also assumed dollars without source currency; that recurrence supports a generic units/provenance experiment rather than a scenario-specific formatting patch. Validate receipt-based lifecycle completion and source-preserving units on structured data, invoices and unrelated document workflows.\n\n## Last three same-scenario trials\n\n- [craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/facts.json): fail, fixed 4.6, 448.5 seconds, 114.45 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: no step of craftbook \"regional-sales-rollup\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region), out/north-summary.md is 0 bytes, need ≥ 80, out/south-summary.md is 0 bytes, need ≥ 80. Last failure: no step of craftbook \"regional-sales-rollup\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes \n- [craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr/facts.json): fail, fixed 5, 476.8 seconds, 110.25 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: no step of craftbook \"regional-sales-summary\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not hardcoded to one region), out/south-summary.md is 0 bytes, need ≥ 80. Last failure: no step of craftbook \"regional-sales-summary\" refers to its input parameter — the region has to reach the steps (the export it reads and the summary it writes must be derived from the parameter, not ha\n- [craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-29-11-494Z-gne7/facts.json): fail, fixed 5, 427.1 seconds, 117.38 recorded tokens/sec. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: regional sales rollup; missing signals stayed unchanged: out/north-summary.md is 0 bytes, need ≥ 80, out/south-summary.md is 0 bytes, need ≥ 80. Last failure: out/north-summary.md is 0 bytes, need ≥ 80\n\nEarlier outcomes: 0/3 passed. Different model/native/source identities mean these are historical context, not controlled native-version comparisons.\n\n## Strategic note\n\nA fixed pass here verifies template/parameter routing and output presence/distinctness. Independent review additionally establishes correct values, but the final state still contains an active south task. Report those three evidence levels separately. No tuning, prompt, gate, source or model was changed and no capability failure retried.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-workflow-audit.json and historical-comparison.json.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "scenarioId": "craftbook-author-gate-script", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T03:15:52.590Z", "finishedAt": "2026-11-05T05:56:40.701Z", "durationMs": 9648111, "success": true, "reason": "authored a craftbook with a substantive embedded gate script, the report parses, and the task completed", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-gate-script\\craftbook-author-gate-script\\craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "scenarioId": "craftbook-author-gate-script", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 3267245 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 5, "scoreMax": 5, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.609707007808811 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 113 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 3267245 }, { "source": "facts", "path": "outcome.durationMs", "value": 9648111 } ] }, "behavior": { "score": 7.5, "ruleId": "B_TWO_INTERVENTIONS", "summary": "No red flags or missing roles were observed; two interventions receive the midpoint between the fixed low/high-intervention anchors.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 2 } ] } }, "composite": 8.1, "band": "ship-ready" }, "facts": { "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "scenarioId": "craftbook-author-gate-script", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-gate-script\\craftbook-author-gate-script\\craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 1623, "peakRssMb": 16959, "peakCpuPercent": 1325.4, "firstAtMs": 0, "lastAtMs": 9635731 }, "gpu": { "available": true, "sampleCount": 1623, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1623, "peakUsedMb": 31486, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 462854, "totalOutputTokens": 60666, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 184488, "todayTokensCached": 147737, "todayTokensOut": 8653, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 184488, "totalTokensCached": 147737, "totalTokensOut": 8653, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 1105, "cachedInputTokens": 344, "outputTokens": 193, "cost": 0, "durationMs": 24888, "outputTokensPerSec": 8.71713430384398, "at": "2026-11-05T05:48:24.102Z" }, "lastUpdated": "2026-11-05T05:48:24.102Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T05:56:36.048Z" } }, "source": "engine-log" }, "derived": { "durationMs": 9637349, "meanTokensPerSec": 8.31, "genTokensPerSec": 8.31, "genTokens": 60666, "promptTokensPerSec": 300.4, "peakRssMb": 16959, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 184488, "output": 8653 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 6, "redrives": 0, "perStep": [ { "ref": "inventory-health-check/1", "stepId": "build", "activatedAt": "2026-11-05T04:05:51.660Z", "completedAt": null, "ms": null, "gateRejections": 6, "toolCalls": 31, "sessionIds": [ "283807d7-8c55-46f8-b599-363fac448e53" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T03:15:58.329Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T03:15:58.231Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "reza": 1, "thiago": 1 }, "perTask": { "inventory-health-check/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 11, "maxContextFill": 0.5007781982421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 2, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5007781982421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 8 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/project-type/inventory-reorder", "knowledge://bendyline/handboek/craftbook/status-report", "knowledge://bendyline/handboek/craftbook/browser-qa-audit", "knowledge://bendyline/handboek/craftbook/data-quality-audit" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "authored a craftbook with a substantive embedded gate script, the report parses, and the task completed", "durationMs": 9648111, "timeoutMs": 15824176, "budgetUsedFraction": 0.609707007808811 }, "timing": { "startedAt": "2026-11-05T03:15:52.590Z", "finishedAt": "2026-11-05T05:56:40.701Z", "timeToFirstArtifactMs": 3267245, "timeToLastArtifactWriteMs": 5284480, "timeToFirstTokenMs": 46936, "firstTurnTtftMs": 36847, "timeToFirstToolCallMs": 57098 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 113, "byTool": { "list_craftbooks": 2, "read_file": 7, "craftbook_read": 8, "how_do_i": 6, "search": 28, "read_document": 9, "craftbook_write": 3, "invoke_craftbook": 1, "list_tasks": 2, "read_task_notes": 13, "get_task": 2, "stat": 2, "advance_task_step": 8, "set_task_status": 17, "write_task_note": 1, "ask_user_question": 1, "write_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 3 }, "sniff": { "progression": [ { "atMs": 7671, "filePath": "craftbook-author-gate-script", "bytes": 0, "score": 3, "scoreMax": 5, "signals": [], "failReason": "no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template" }, { "atMs": 2970353, "filePath": "craftbook-author-gate-script", "bytes": 2464, "score": 3, "scoreMax": 5, "signals": [], "failReason": "no task has been created from craftbook \"inventory-health-report\" yet — invoke the craftbook on this project" }, { "atMs": 2999883, "filePath": "craftbook-author-gate-script", "bytes": 2464, "score": 3, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3272757, "filePath": "craftbook-author-gate-script", "bytes": 3153, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3356116, "filePath": "craftbook-author-gate-script", "bytes": 3153, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3362145, "filePath": "craftbook-author-gate-script", "bytes": 3153, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3415422, "filePath": "craftbook-author-gate-script", "bytes": 3153, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3474808, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3623415, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3629334, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3712923, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3718681, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 4295452, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 4307302, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 8770306, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 8799988, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 9643003, "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 5, "scoreMax": 5, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-author-gate-script", "bytes": 2989, "score": 5, "scoreMax": 5, "signals": [], "failReason": null } }, "autoAnswer": { "total": 2, "byKind": { "structured": 2, "inline": 0 }, "events": [ { "atMs": 8693092, "kind": "structured", "gezel": "reza", "question": "The inventory report file (out/inventory-report.json) is already written and cor…", "chose": "Resume the task now (I'll try to advance it — but the gate will likely hit the same sandbox error again)" }, { "atMs": 9642985, "kind": "structured", "gezel": "reza", "question": "Your file is finished. - **`out/inventory-report.json`**", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-05T03:15:52.593Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-gate-script-p3cBG7", "[2026-11-05T03:15:52.593Z] [trial] id=craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt scenario=craftbook-author-gate-script model=underdog-saluki-27b-iq2-mix", "[2026-11-05T03:15:52.593Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T03:15:52.593Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-gate-script\\craftbook-author-gate-script\\craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "[2026-11-05T03:15:52.596Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T03:15:52.594Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T03:15:52.600Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T03:15:52.606Z] [trial] throughput-scaled ceiling: 120m → 264m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T03:15:52.606Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-gate-script-p3cBG7", "[2026-11-05T03:15:58.327Z] [trial] daemon spawned pid=50444 port=https://127.0.0.1:53460", "[2026-11-05T03:15:59.661Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T03:15:59.680Z] [trial] meester=yusuf", "[2026-11-05T03:15:59.680Z] [trial] running scenario setup hook", "[2026-11-05T03:16:00.050Z] [trial] scenario setup complete", "[2026-11-05T03:16:00.052Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T03:16:00.114Z] [poll] initial digests hard=5e3f0bc905c75f4b soft=385de44f6091e413", "[2026-11-05T03:16:00.389Z] [poll] hard-progress: 5e3f0bc905c75f4b → 98aac5d9d7ea71aa after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=3/0B)", "[2026-11-05T03:16:52.959Z] [poll] hard-progress: 98aac5d9d7ea71aa → 6cfcd0930715f2ed after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=37,stream=67) sniff=3/0B)", "[2026-11-05T03:17:22.575Z] [poll] hard-progress: 6cfcd0930715f2ed → d1fece7e9d681f90 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=82,stream=190) sniff=3/0B)", "[2026-11-05T03:18:09.717Z] [poll] hard-progress: d1fece7e9d681f90 → e0e1634cbc095a79 after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=164,stream=448) sniff=3/0B)", "[2026-11-05T03:19:03.185Z] [poll] hard-progress: e0e1634cbc095a79 → 891fc90737a42a94 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=200,stream=555) sniff=3/0B)", "[2026-11-05T03:20:14.096Z] [poll] hard-progress: 891fc90737a42a94 → ce24b275974ef7e3 after 71s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=236,stream=647) sniff=3/0B)", "[2026-11-05T03:20:38.776Z] [poll] hard-progress: ce24b275974ef7e3 → a269a40a6bea6747 after 25s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=273,stream=754) sniff=3/0B)", "[2026-11-05T03:21:43.911Z] [poll] hard-progress: a269a40a6bea6747 → d6b0e3affe146695 after 65s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=320,stream=894) sniff=3/0B)", "[2026-11-05T03:22:07.581Z] [poll] hard-progress: d6b0e3affe146695 → bfa775d83c3fb95e after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=355,stream=1009) sniff=3/0B)" ] }, "metrics": { "process": { "sampleCount": 1623, "peakRssMb": 16959, "peakCpuPercent": 1325.4, "firstAtMs": 0, "lastAtMs": 9635731 }, "gpu": { "available": true, "sampleCount": 1623, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1623, "peakUsedMb": 31486, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 462854, "totalOutputTokens": 60666, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 184488, "todayTokensCached": 147737, "todayTokensOut": 8653, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 184488, "totalTokensCached": 147737, "totalTokensOut": 8653, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 1105, "cachedInputTokens": 344, "outputTokens": 193, "cost": 0, "durationMs": 24888, "outputTokensPerSec": 8.71713430384398, "at": "2026-11-05T05:48:24.102Z" }, "lastUpdated": "2026-11-05T05:48:24.102Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T05:56:36.048Z" } }, "source": "engine-log" }, "derived": { "durationMs": 9637349, "meanTokensPerSec": 8.31, "genTokensPerSec": 8.31, "genTokens": 60666, "promptTokensPerSec": 300.4, "peakRssMb": 16959, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 184488, "output": 8653 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T03:15:52.590Z", "finishedAt": "2026-11-05T05:56:40.701Z", "timeToFirstArtifactMs": 3267245, "timeToLastArtifactWriteMs": 5284480, "timeToFirstTokenMs": 46936, "firstTurnTtftMs": 36847, "timeToFirstToolCallMs": 57098 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 6, "redrives": 0, "perStep": [ { "ref": "inventory-health-check/1", "stepId": "build", "activatedAt": "2026-11-05T04:05:51.660Z", "completedAt": null, "ms": null, "gateRejections": 6, "toolCalls": 31, "sessionIds": [ "283807d7-8c55-46f8-b599-363fac448e53" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T03:15:58.329Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T03:15:58.231Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "reza": 1, "thiago": 1 }, "perTask": { "inventory-health-check/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 11, "maxContextFill": 0.5007781982421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 2, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5007781982421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 8 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "scenarioId": "craftbook-author-gate-script", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/workspace/inventory-health-check/out/inventory-report.json", "artifactBytes": 525, "artifactSha256": "c6f1ae25c1892003c66b44c030cf8085ef72e0d161c0f2aa8c9fd9bcfc57d87d", "axisSource": "evals/src/craftbooks/authoring/gate-script.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. Counts and anomaly lists are correct, but totalValue is 10.00 too low. Fixed pass came from set_task_status complete while still on build: no successful custom-gate run or verify step. A verified sandbox dependency was restored during this attempt; the model then closed on prose that explicitly acknowledged the gate had not run.", "reviewBasis": "Read final JSON, source inventory, saved task/recipe, exact close call, six failed script runs and all post-repair tool events. Independently summed integer cents. Compared with prior checkpoints and recorded dependency repair; kept fixed score unchanged.", "environmentRepair": "evals/runs/saluki-native-0.1.57-2026-11-03/mxc-home-repair.json", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/workspace/inventory-health-check/out/inventory-report.json", "bytes": 525, "sha256": "c6f1ae25c1892003c66b44c030cf8085ef72e0d161c0f2aa8c9fd9bcfc57d87d" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T16:59:59.806Z", "reviewId": "042", "cellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "scenarioId": "craftbook-author-gate-script", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b1301213819175f0798ea1381907ad040366c58c08d048bd74c81ae6a64f5e26", "evidenceSha256": "9ff3acc79566b4a3efee6476d1ee0284dcce05b436cf7715f835a1ff566a7994", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/workspace/inventory-health-check/out/inventory-report.json", "bytes": 525, "sha256": "c6f1ae25c1892003c66b44c030cf8085ef72e0d161c0f2aa8c9fd9bcfc57d87d", "shownBytes": 525, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-gate-script\\craftbook-author-gate-script\\craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt\\workspace\\inventory-health-check\\data\\inventory.json", "bytes": 1006, "sha256": "2a6f44cb0bb8d193c2abf527264e828fbf882e8a744180a786e784fe0688b439", "shownBytes": 1006, "truncated": false } ], "scoreAxes": {}, "justification": "The output report matches the inventory: 10 items, 583 units, 2 out of stock, 1 negative (SKU-509), 1 low stock (SKU-707); totalValue looks plausible. The craftbook, its inline gate script, the invocation and the verification step are not in the evidence, so they were not assessed.", "limitations": "The craftbook document, gate script (SDK import, gateResult output), the order of craftbook_write before delegation, the invoke_craftbook call and the verification step are not shown. I did not recompute totalValue or the low-stock threshold. The evidence shows only the final report.", "meanScore": null, "durationMs": 14158, "costUsd": 0.016112, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3227, "cache_read_input_tokens": 700, "output_tokens": 313, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3227, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 313, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3227, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3227 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 313, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3227, "webSearchRequests": 0, "costUSD": 0.016112, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3227, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-gate-script / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.1 / 10** (band: ship-ready)\n**Outcome: success** (duration: 160m 48s; budget used: 61.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 3267245`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":2989,\"score\":5,\"scoreMax\":5,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.609707007808811`; `facts.toolUse.totalToolCalls: 113`; `facts.timing.timeToFirstArtifactMs: 3267245`; `facts.outcome.durationMs: 9648111` |\n| Behavior soundness (15%) | 7.5 / 10 | No red flags or missing roles were observed; two interventions receive the midpoint between the fixed low/high-intervention anchors. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 2` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 7.5 = **8.1**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 16959 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31486 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.31 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 462854 + 60666 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 6 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 2 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 50% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 8 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 3267245`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":2989,\"score\":5,\"scoreMax\":5,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.609707007808811`\n- `facts.toolUse.totalToolCalls: 113`\n- `facts.timing.timeToFirstArtifactMs: 3267245`\n- `facts.outcome.durationMs: 9648111`\n\n### Behavior soundness - B_TWO_INTERVENTIONS\n\nNo red flags or missing roles were observed; two interventions receive the midpoint between the fixed low/high-intervention anchors.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 2`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 3267245`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":2989,\"score\":5,\"scoreMax\":5,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.609707007808811`; `facts.toolUse.totalToolCalls: 113`; `facts.timing.timeToFirstArtifactMs: 3267245`; `facts.outcome.durationMs: 9648111`\n- Behavior: No red flags or missing roles were observed; two interventions receive the midpoint between the fixed low/high-intervention anchors. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 2`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\n**Fixed pass, 8.1/10, preserved.** This scenario declares no judge axes; codex-inline/current-session review is prose only. The recorded pass does not demonstrate successful custom-gate execution.\n\nNo declared axes. Counts and anomaly lists are correct, but totalValue is 10.00 too low. Fixed pass came from set_task_status complete while still on build: no successful custom-gate run or verify step. A verified sandbox dependency was restored during this attempt; the model then closed on prose that explicitly acknowledged the gate had not run.\n\n## Positives\n\n- A reusable two-step recipe with a substantive embedded script was saved and invoked once. Its gate imports SDK checks and checks required report sections, inventory count, and zero/negative SKU coverage (toolUse.byTool.craftbook_write: 3; invoke_craftbook: 1; sniff.latest.score: 5/5; final task.craftbook).\n- The 525-byte final JSON contains the correct ten-item count, 583 signed units, seven positive-stock items, two zero-stock SKUs, one negative-stock SKU, and one low-stock SKU under the declared 0 < qty <= 10 policy (sniff.latest artifact evidence; toolUse.byTool.write_file: 3; advisory sourceAudit).\n- One CUDA/native 0.1.57 launch ran through 60666 generated tokens with no explicit OOM, context overflow or unexpected native exit in the captured log. Mean decode was 8.31 tokens/sec (perf.usage.totalOutputTokens; perf.derived.meanTokensPerSec; result.engineContext.launches).\n\n## Negatives\n\n- The successful close at 05:56:33.365Z is set_task_status complete with verification text admitting the custom gate could not run. Final task remains on build, with no step completion timestamp; verify was never activated. All six script-run records are infrastructure failures before the repair, and no advance_task_step occurs after it. Fixed outcome.success is true, but this is not evidence that the gate or two-step recipe executed successfully (outcome.success; toolUse.byTool.set_task_status: 17; continuity.steps.completed: 0; close-call and state evidence).\n- Final totalValue is 7375.60 rather than 7385.60 from the source sum of signed quantity times unit price. The close prose repeats the incorrect value and calls it verified. The authored gate omits value/unit totals, category counts, and low-stock content checks; its verify prompt omits value too (sniff.latest full gate; toolUse.byTool.write_file: 3; final report/template and independent sourceAudit).\n- The gate calls gezel.output in rejection branches without ending evaluation; the SDK function returns void and permits only one stamp. Missing/invalid inputs could continue to a second output call or throw. This is a static error-path finding, not an observed successful execution (toolUse.byTool.craftbook_write: 3; saved script; packages/sdk/src/portable.ts).\n- Fresh trial homes suppressed bootstrap and lacked the verified MXC launcher. Six gate attempts failed closed on the absent denyNet boundary. A recorded environment repair copied the existing pinned launcher and original sentinel into this disposable home, and the live daemon boundary probe passed at 05:43:39.516Z. Treat elapsed time and completion as infrastructure-assisted evidence; no model message, task mutation, gate bypass or capability rerun was performed by Codex (outcome.durationMs; toolUse.byTool.advance_task_step: 8; daemon script-run records; mxc-home-repair.json).\n- The 160m48s trial consumed 113 tool calls, including 28 search, 13 read_task_notes and 17 set_task_status calls, while infrastructure pauses were treated as a repairable content plateau. Two structured auto-answers and eight tool-repeat aborts show sustained lifecycle churn (toolUse.byTool/totalToolCalls; autoAnswer.total: 2; continuity.budget.toolRepeatAborts: 8).\n\n## Tactical experiment — not applied\n\nIn AUTHORING_WORKER_ABOUT in evals/src/craftbooks/authoring/helpers.ts, add a general closure rule: an unavailable execution check is an infrastructure blocker; file presence and a prose assertion do not establish that the check ran. Require completion claims to name the actual check result, and keep calculated totals distinct from unchecked estimates. Measure unsupported success claims on report gates and unrelated developer workflows. This supports the service invariant below; it must not substitute for enforcing that invariant.\n\n## Strategic experiments — not applied\n\n1. Enforce gated closure at the service boundary in packages/service/src/tasks/manager.ts setStatusOnce and packages/service/src/http/routes/project-tasks.ts. A model-origin complete request must not close a paused nonterminal step with an unmet gate. Route it through the existing step completion machinery or reject it with evidence. The MCP check in packages/mcp/src/server.ts currently accepts nonempty verification prose and a deliverable; it cannot establish gate execution. Validate paused, active and terminal closure across script, reviewer and code-execution gates.\n2. In evals/src/craftbooks/authoring/gate-script.ts, record successful execution of the declared custom gate and terminal verification separately from task.status. Its current status-plus-file condition accepted this close. Preserve frozen scores here; validate any future stricter evaluator on known passing and held tasks.\n3. Resolve a verified offline MXC dependency during Windows eval preflight in evals/src/spawn.ts / scorecard-campaign preflight; do not bypass denyNet. Propagate gate_infrastructure through finishAuthoringPoll in authoring/helpers.ts so a runtime failure is not repeatedly sent to a model as a document defect.\n4. Document one-output/early-return gate patterns in SDK examples and authoring guidance. Cover missing files, malformed JSON, mismatched aggregates and success; recompute report values from source integer cents in future gates.\n\n## Performance and GPU fit\n\nElapsed160m48s of the normal throughput-scaled264m ceiling; recorded462854 input and60666 output tokens, mean decode8.31tok/s, prefill300.4tok/s. Peak VRAM10646/12227MiB; process-tree RSS16959MiB (new campaign high); whole-system RAM31486/31865MiB. One native0.1.57 CUDA launch,65536-token context, one slot, f16 KV, reasoning4096. Dense FFN from39/64 layers on GPU,25 on CPU; shared-GPU-memory/paging unmeasured. Two mid-turn compactions, no recorded context overflow or OOM. Infrastructure delay is retained in elapsed time and not interpreted as raw inference throughput.\n\n## Generalization check and next experiment\n\nThe missing boundary is a host dependency issue, while closing a gated task from a paused build step is a general lifecycle invariant failure. Author-params also obtained a fixed pass before both tasks were terminal, and invoice-run showed repeated validator/router friction. These independent cases support checking actual receipts and lifecycle transitions across tasks, rather than trusting a terminal label. First test the service closure invariant on custom-script and unrelated code-review tasks; expect zero unsupported completion transitions while legitimate terminal closes remain possible.\n\n## Last three same-scenario trials\n\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/facts.json): fail, fixed4.5, 43.1min. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: craftbook \"inventory-health-check\" has no step gated by a scope:\"craftbook\" script — attach the custom check script to the build step's gate. Last failure: craftbook \"inventory-health-check\" has no step gated by a scope:\"craftbook\" script — attach the custom check script to the build step's gate\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f/facts.json): fail, fixed5.2, 5.2min. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: craftbook \"inventory-health-check\" step \"build\" references gate script \"verifyInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself, task inventory-health-check/1 (from craftbook \"inventory-health-check\") has status \"paused\" — advance THIS task to a terminal step with advan\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T18-51-49-809Z-so5p/facts.json): fail, fixed5.2, 10.6min. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template. Last failure: no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template\n\nAll three earlier trials failed. Their different model/native/source identities prevent attributing this fixed-score increase to native0.1.57. The current status-only pass and operator dependency repair especially limit that comparison.\n\n## Strategic note\n\nDo not count this as demonstrated self-checking workflow competence. The fixed rubric reports8.1/pass; the advisory evidence establishes a structured report with one arithmetic error, a plausible but unexecuted gate, and an accepted model-driven closure while still on build. The standard campaign after-trial identity check recorded no comparisonIssue and proceeded. No score, source, prompt, gate or tuning was altered; no failed capability cell was rerun.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-workflow-audit.json, advisory-artifact-review.json, historical-comparison.json, and the campaign mxc-home-repair.json.\n" }, { "cellId": "r1-saluki-cuda-complex-work-craftbook-author-fanout", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-fanout", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 1, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "scenarioId": "craftbook-author-fanout", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T06:01:02.972Z", "finishedAt": "2026-11-05T09:47:26.889Z", "durationMs": 13583917, "success": true, "reason": "authored a single fanout craftbook over data/stores.json, one run produced a unit of work per store, and all 4 write-ups exist", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-fanout\\craftbook-author-fanout\\craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "scenarioId": "craftbook-author-fanout", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1209655 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-fanout", "bytes": 4047, "score": 8, "scoreMax": 8, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.8584280786563547 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 217 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1209655 }, { "source": "facts", "path": "outcome.durationMs", "value": 13583917 } ] }, "behavior": { "score": 7.5, "ruleId": "B_TWO_INTERVENTIONS", "summary": "No red flags or missing roles were observed; two interventions receive the midpoint between the fixed low/high-intervention anchors.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 2 } ] } }, "composite": 8.1, "band": "ship-ready" }, "facts": { "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "scenarioId": "craftbook-author-fanout", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-fanout\\craftbook-author-fanout\\craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 2283, "peakRssMb": 15539, "peakCpuPercent": 1398.2, "firstAtMs": 0, "lastAtMs": 13571614 }, "gpu": { "available": true, "sampleCount": 2283, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 2283, "peakUsedMb": 30882, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 671221, "totalOutputTokens": 94025, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 23, "todayTokensIn": 412972, "todayTokensCached": 377721, "todayTokensOut": 4415, "todayCost": 0, "totalTurns": 23, "totalTokensIn": 412972, "totalTokensCached": 377721, "totalTokensOut": 4415, "totalCost": 0, "medianOutputTokensPerSec": 9.1, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.1, "turns": 23 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10979, "cachedInputTokens": 10615, "outputTokens": 127, "cost": 0, "durationMs": 682678, "outputTokensPerSec": 9.178242117510782, "at": "2026-11-05T09:32:11.340Z" }, "lastUpdated": "2026-11-05T09:32:11.340Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T09:47:22.335Z" } }, "source": "engine-log" }, "derived": { "durationMs": 13573369, "meanTokensPerSec": 8.74, "genTokensPerSec": 8.74, "genTokens": 94025, "promptTokensPerSec": 320.92, "peakRssMb": 15539, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 412972, "output": 4415 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 6 }, "steps": { "activated": 19, "activationEvents": 9, "completed": 15, "gateApprovals": 12, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "store-health-sweep/1", "stepId": "load", "activatedAt": "2026-11-05T06:15:27.843Z", "completedAt": "2026-11-05T06:21:12.591Z", "ms": 344748, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "d820077a-ad26-49d4-8e9d-12ab7e83fbfe" ] }, { "ref": "store-health-sweep/1", "stepId": "write-ups", "activatedAt": "2026-11-05T06:21:12.592Z", "completedAt": "2026-11-05T06:34:16.194Z", "ms": 783602, "gateRejections": 0, "toolCalls": 13, "sessionIds": [ "1dc99522-f587-4b42-8a16-7fb5a7d54c9a" ] }, { "ref": "store-health-sweep/1", "stepId": "verify", "activatedAt": "2026-11-05T06:34:16.195Z", "completedAt": "2026-11-05T06:44:57.993Z", "ms": 641798, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "4d2cbc4e-3962-49ae-9d52-72429fb3d5c2" ] }, { "ref": "store-health-sweep/1", "stepId": "finish", "activatedAt": "2026-11-05T06:44:57.994Z", "completedAt": "2026-11-05T06:50:15.227Z", "ms": 317233, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "e7ce9618-58cf-4114-b9ff-385a68d44728" ] }, { "ref": "store-health-sweep/2", "stepId": "load", "activatedAt": "2026-11-05T07:05:18.984Z", "completedAt": "2026-11-05T07:13:01.813Z", "ms": 462829, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "624f7473-2b20-48a7-a983-72f0ed1ff916" ] }, { "ref": "store-health-sweep/2", "stepId": "fanout", "activatedAt": "2026-11-05T07:13:01.815Z", "completedAt": "2026-11-05T07:41:02.703Z", "ms": 1680888, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "f4517889-ff98-4e22-aa67-a81a6b41090b" ] }, { "ref": "store-health-sweep/2", "stepId": "verify", "activatedAt": "2026-11-05T07:41:02.704Z", "completedAt": "2026-11-05T08:34:54.909Z", "ms": 3232205, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "5dfcc353-080e-48e8-bfc6-ea4ce29b5e2e" ] }, { "ref": "store-health-sweep/2", "stepId": "finish", "activatedAt": "2026-11-05T08:34:54.911Z", "completedAt": "2026-11-05T08:52:14.215Z", "ms": 1039304, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "24e55a99-f410-43bc-a85c-35755e8865ae" ] }, { "ref": "store-health-sweep/3", "stepId": "load", "activatedAt": "2026-11-05T08:33:01.865Z", "completedAt": "2026-11-05T08:48:33.769Z", "ms": 931904, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "27a32365-e947-42b7-8161-5dcf42ed5661" ] }, { "ref": "store-health-sweep/3", "stepId": "fanout", "activatedAt": "2026-11-05T08:48:33.771Z", "completedAt": "2026-11-05T08:48:34.044Z", "ms": 273, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "store-health-sweep/3", "stepId": "verify", "activatedAt": "2026-11-05T08:48:34.045Z", "completedAt": "2026-11-05T09:32:11.282Z", "ms": 2617237, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "b1508d05-c33d-4cf4-8a1c-89c2d0a35df0" ] }, { "ref": "store-health-sweep/4", "stepId": "write-health", "activatedAt": "2026-11-05T08:48:33.790Z", "completedAt": "2026-11-05T08:58:11.859Z", "ms": 578069, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "8c7a6d3b-2776-4383-a8e1-661926a66d45" ] }, { "ref": "store-health-sweep/5", "stepId": "write-health", "activatedAt": "2026-11-05T08:48:33.851Z", "completedAt": "2026-11-05T09:05:18.945Z", "ms": 1005094, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "b336cf37-752f-471e-b90b-71feec455389" ] }, { "ref": "store-health-sweep/6", "stepId": "write-health", "activatedAt": "2026-11-05T08:48:33.897Z", "completedAt": "2026-11-05T09:12:28.366Z", "ms": 1434469, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "29c867f7-8b1e-41f2-9496-13ea2dc3fd75" ] }, { "ref": "store-health-sweep/7", "stepId": "write-health", "activatedAt": "2026-11-05T08:48:33.939Z", "completedAt": "2026-11-05T09:19:44.954Z", "ms": 1871015, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "9ad4112c-1ef4-4f11-bc7a-526d72c1d110" ] }, { "ref": "store-health-sweep/3", "stepId": "finish", "activatedAt": "2026-11-05T09:32:11.284Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "bd82bc9e-2729-4369-9ceb-c406b598432c" ] }, { "ref": "store-health-sweep/8", "stepId": "load", "activatedAt": "2026-11-05T09:47:17.643Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T06:01:08.914Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T06:01:08.830Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 931904 }, "sessions": { "total": 17, "taskScoped": 15, "byGezel": { "darius": 1, "frida": 3, "kebba": 8, "reza": 1, "viorica": 4 }, "perTask": { "store-health-sweep/2": 4, "store-health-sweep/1": 4, "store-health-sweep/3": 3, "store-health-sweep/6": 1, "store-health-sweep/4": 1, "store-health-sweep/7": 1, "store-health-sweep/5": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7894736842105263, "maxMessages": 19, "maxContextFill": 0.5356903076171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 4, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5356903076171875 }, "fanout": { "hosts": 2, "childrenSpawned": 4, "childrenCompleted": 4, "childrenFailed": 0, "barrierHolds": 1, "barrierReleases": 1, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 4, "tasks": 10, "tasksWithReferences": 8, "items": 40, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbooks-overview", "knowledge://bendyline/handboek/craftbook/cookbook-layout", "knowledge://bendyline/handboek/craftbook/seasonal-maintenance-sweep", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/script-automation", "knowledge://bendyline/handboek/craftbook/nightly-fix-sweep", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/craftbook/holiday-card-run", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/script-automation", "knowledge://bendyline/handboek/craftbook/nightly-fix-sweep", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/craftbook/holiday-card-run", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/script-automation", "knowledge://bendyline/handboek/craftbook/nightly-fix-sweep", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/craftbook/holiday-card-run", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/script-automation", "knowledge://bendyline/handboek/craftbook/nightly-fix-sweep", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/craftbook/holiday-card-run", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/script-automation", "knowledge://bendyline/handboek/craftbook/nightly-fix-sweep", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/craftbook/holiday-card-run", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/script-automation", "knowledge://bendyline/handboek/craftbook/nightly-fix-sweep", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/craftbook/holiday-card-run", "shared:About this library.md", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/game-with-screens", "knowledge://bendyline/handboek/craftbook/holiday-card-run", "knowledge://bendyline/handboek/craftbook/scrape-to-structured" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "authored a single fanout craftbook over data/stores.json, one run produced a unit of work per store, and all 4 write-ups exist", "durationMs": 13583917, "timeoutMs": 15824176, "budgetUsedFraction": 0.8584280786563547 }, "timing": { "startedAt": "2026-11-05T06:01:02.972Z", "finishedAt": "2026-11-05T09:47:26.889Z", "timeToFirstArtifactMs": 1209655, "timeToLastArtifactWriteMs": 12668369, "timeToFirstTokenMs": 48878, "firstTurnTtftMs": 37199, "timeToFirstToolCallMs": 58298 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Data-Analyst", "Copywriter", "Reviewer", "Planner" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 217, "byTool": { "read_file": 18, "list_craftbooks": 5, "craftbook_read": 18, "craftbook_write": 8, "invoke_craftbook": 5, "read_task_notes": 13, "find_files": 3, "get_task": 11, "list_artifacts": 4, "message_gezel": 4, "stat": 9, "list_dir": 9, "read_files": 10, "write_task_note": 12, "advance_task_step": 14, "how_do_i": 12, "search": 2, "list_tasks": 3, "list_task_children": 5, "set_task_status": 1, "read_artifact": 7, "write_artifact": 20, "write_file": 20, "validate": 4 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 18 }, "sniff": { "progression": [ { "atMs": 7739, "filePath": "craftbook-author-fanout", "bytes": 0, "score": 3, "scoreMax": 8, "signals": [], "failReason": "no reusable authored craftbook exists yet — author the store health sweep as a craftbook template, not as ad-hoc chat work" }, { "atMs": 832801, "filePath": "craftbook-author-fanout", "bytes": 38, "score": 2, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "atMs": 868453, "filePath": "craftbook-author-fanout", "bytes": 38, "score": 2, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "atMs": 1463948, "filePath": "craftbook-author-fanout", "bytes": 641, "score": 4, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "atMs": 1576547, "filePath": "craftbook-author-fanout", "bytes": 1492, "score": 6, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "atMs": 1831423, "filePath": "craftbook-author-fanout", "bytes": 1442, "score": 6, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "atMs": 2103914, "filePath": "craftbook-author-fanout", "bytes": 1549, "score": 6, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "atMs": 3831162, "filePath": "craftbook-author-fanout", "bytes": 1546, "score": 6, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "atMs": 3860780, "filePath": "craftbook-author-fanout", "bytes": 1546, "score": 6, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store" }, { "atMs": 8899158, "filePath": "craftbook-author-fanout", "bytes": 3450, "score": 7, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" has produced 2 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run" }, { "atMs": 9124874, "filePath": "craftbook-author-fanout", "bytes": 3450, "score": 7, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" has produced 3 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run" }, { "atMs": 10478644, "filePath": "craftbook-author-fanout", "bytes": 3639, "score": 7, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" has produced 3 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run" }, { "atMs": 10851793, "filePath": "craftbook-author-fanout", "bytes": 3803, "score": 7, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" has produced 3 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run" }, { "atMs": 11409648, "filePath": "craftbook-author-fanout", "bytes": 3887, "score": 7, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" has produced 3 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run" }, { "atMs": 11664685, "filePath": "craftbook-author-fanout", "bytes": 4083, "score": 7, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" has produced 3 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run" }, { "atMs": 13335438, "filePath": "craftbook-author-fanout", "bytes": 4047, "score": 7, "scoreMax": 8, "signals": [], "failReason": "craftbook \"store-health-sweep\" has produced 3 of 4 per-store runs — invoke the recipe on this project and make sure its fanout step really fans out over data/stores.json, so each store gets its own run" }, { "atMs": 13578918, "filePath": "craftbook-author-fanout", "bytes": 4047, "score": 8, "scoreMax": 8, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-author-fanout", "bytes": 4047, "score": 8, "scoreMax": 8, "signals": [], "failReason": null } }, "autoAnswer": { "total": 2, "byKind": { "structured": 2, "inline": 0 }, "events": [ { "atMs": 2953309, "kind": "structured", "gezel": "reza", "question": "**Store Health Sweep — all stores in data/stores.json** is finished. - **`tasks…", "chose": "Dismiss" }, { "atMs": 10276408, "kind": "structured", "gezel": "reza", "question": "Your document is finished. - **`tasks/2/fanout.md`** - `tasks/2/verdict.md` - `…", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-05T06:01:02.976Z] [trial] id=craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04 scenario=craftbook-author-fanout model=underdog-saluki-27b-iq2-mix", "[2026-11-05T06:01:02.976Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-fanout\\craftbook-author-fanout\\craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "[2026-11-05T06:01:02.976Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T06:01:02.976Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-fanout-JY6W70", "[2026-11-05T06:01:02.976Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T06:01:02.979Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T06:01:02.983Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T06:01:02.990Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-fanout-JY6W70", "[2026-11-05T06:01:02.990Z] [trial] throughput-scaled ceiling: 120m → 264m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T06:01:08.814Z] [trial] daemon spawned pid=31764 port=https://127.0.0.1:50171", "[2026-11-05T06:01:10.110Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T06:01:10.127Z] [trial] meester=cosima", "[2026-11-05T06:01:10.127Z] [trial] running scenario setup hook", "[2026-11-05T06:01:10.465Z] [trial] scenario setup complete", "[2026-11-05T06:01:10.466Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T06:01:10.530Z] [poll] initial digests hard=30c2aa027a6044c6 soft=dcb3116d4216e53f", "[2026-11-05T06:01:10.880Z] [poll] hard-progress: 30c2aa027a6044c6 → 9ced21a907720066 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=3/0B)", "[2026-11-05T06:02:03.402Z] [poll] hard-progress: 9ced21a907720066 → f1574b71acbddc24 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=37,stream=61) sniff=3/0B)", "[2026-11-05T06:02:21.122Z] [poll] hard-progress: f1574b71acbddc24 → 2647003d0e4a1279 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=58,stream=111) sniff=3/0B)", "[2026-11-05T06:14:55.914Z] [poll] hard-progress: 2647003d0e4a1279 → 9b8fbf7bee0bddd4 after 755s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=2159,stream=8234) sniff=2/38B)", "[2026-11-05T06:15:31.549Z] [poll] hard-progress: 9b8fbf7bee0bddd4 → 857025dae22354ca after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=2237,stream=8610) sniff=2/38B)", "[2026-11-05T06:16:31.049Z] [poll] hard-progress: 857025dae22354ca → e6b8ee1837f05e39 after 60s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=5,slot=2346,stream=8917) sniff=2/38B)", "[2026-11-05T06:16:55.323Z] [poll] hard-progress: e6b8ee1837f05e39 → cf2157a5183c5725 after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=2374,stream=8996) sniff=2/38B)", "[2026-11-05T06:17:30.937Z] [poll] hard-progress: cf2157a5183c5725 → 3799b97af54f8b86 after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=2391,stream=9038) sniff=2/38B)", "[2026-11-05T06:17:36.856Z] [poll] hard-progress: 3799b97af54f8b86 → 7d81e0b0a0d2cd6d after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=2404,stream=9073) sniff=2/38B)" ] }, "metrics": { "process": { "sampleCount": 2283, "peakRssMb": 15539, "peakCpuPercent": 1398.2, "firstAtMs": 0, "lastAtMs": 13571614 }, "gpu": { "available": true, "sampleCount": 2283, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 2283, "peakUsedMb": 30882, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 671221, "totalOutputTokens": 94025, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 23, "todayTokensIn": 412972, "todayTokensCached": 377721, "todayTokensOut": 4415, "todayCost": 0, "totalTurns": 23, "totalTokensIn": 412972, "totalTokensCached": 377721, "totalTokensOut": 4415, "totalCost": 0, "medianOutputTokensPerSec": 9.1, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.1, "turns": 23 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10979, "cachedInputTokens": 10615, "outputTokens": 127, "cost": 0, "durationMs": 682678, "outputTokensPerSec": 9.178242117510782, "at": "2026-11-05T09:32:11.340Z" }, "lastUpdated": "2026-11-05T09:32:11.340Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T09:47:22.335Z" } }, "source": "engine-log" }, "derived": { "durationMs": 13573369, "meanTokensPerSec": 8.74, "genTokensPerSec": 8.74, "genTokens": 94025, "promptTokensPerSec": 320.92, "peakRssMb": 15539, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 412972, "output": 4415 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T06:01:02.972Z", "finishedAt": "2026-11-05T09:47:26.889Z", "timeToFirstArtifactMs": 1209655, "timeToLastArtifactWriteMs": 12668369, "timeToFirstTokenMs": 48878, "firstTurnTtftMs": 37199, "timeToFirstToolCallMs": 58298 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 6 }, "steps": { "activated": 19, "activationEvents": 9, "completed": 15, "gateApprovals": 12, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "store-health-sweep/1", "stepId": "load", "activatedAt": "2026-11-05T06:15:27.843Z", "completedAt": "2026-11-05T06:21:12.591Z", "ms": 344748, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "d820077a-ad26-49d4-8e9d-12ab7e83fbfe" ] }, { "ref": "store-health-sweep/1", "stepId": "write-ups", "activatedAt": "2026-11-05T06:21:12.592Z", "completedAt": "2026-11-05T06:34:16.194Z", "ms": 783602, "gateRejections": 0, "toolCalls": 13, "sessionIds": [ "1dc99522-f587-4b42-8a16-7fb5a7d54c9a" ] }, { "ref": "store-health-sweep/1", "stepId": "verify", "activatedAt": "2026-11-05T06:34:16.195Z", "completedAt": "2026-11-05T06:44:57.993Z", "ms": 641798, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "4d2cbc4e-3962-49ae-9d52-72429fb3d5c2" ] }, { "ref": "store-health-sweep/1", "stepId": "finish", "activatedAt": "2026-11-05T06:44:57.994Z", "completedAt": "2026-11-05T06:50:15.227Z", "ms": 317233, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "e7ce9618-58cf-4114-b9ff-385a68d44728" ] }, { "ref": "store-health-sweep/2", "stepId": "load", "activatedAt": "2026-11-05T07:05:18.984Z", "completedAt": "2026-11-05T07:13:01.813Z", "ms": 462829, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "624f7473-2b20-48a7-a983-72f0ed1ff916" ] }, { "ref": "store-health-sweep/2", "stepId": "fanout", "activatedAt": "2026-11-05T07:13:01.815Z", "completedAt": "2026-11-05T07:41:02.703Z", "ms": 1680888, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "f4517889-ff98-4e22-aa67-a81a6b41090b" ] }, { "ref": "store-health-sweep/2", "stepId": "verify", "activatedAt": "2026-11-05T07:41:02.704Z", "completedAt": "2026-11-05T08:34:54.909Z", "ms": 3232205, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "5dfcc353-080e-48e8-bfc6-ea4ce29b5e2e" ] }, { "ref": "store-health-sweep/2", "stepId": "finish", "activatedAt": "2026-11-05T08:34:54.911Z", "completedAt": "2026-11-05T08:52:14.215Z", "ms": 1039304, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "24e55a99-f410-43bc-a85c-35755e8865ae" ] }, { "ref": "store-health-sweep/3", "stepId": "load", "activatedAt": "2026-11-05T08:33:01.865Z", "completedAt": "2026-11-05T08:48:33.769Z", "ms": 931904, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "27a32365-e947-42b7-8161-5dcf42ed5661" ] }, { "ref": "store-health-sweep/3", "stepId": "fanout", "activatedAt": "2026-11-05T08:48:33.771Z", "completedAt": "2026-11-05T08:48:34.044Z", "ms": 273, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "store-health-sweep/3", "stepId": "verify", "activatedAt": "2026-11-05T08:48:34.045Z", "completedAt": "2026-11-05T09:32:11.282Z", "ms": 2617237, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "b1508d05-c33d-4cf4-8a1c-89c2d0a35df0" ] }, { "ref": "store-health-sweep/4", "stepId": "write-health", "activatedAt": "2026-11-05T08:48:33.790Z", "completedAt": "2026-11-05T08:58:11.859Z", "ms": 578069, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "8c7a6d3b-2776-4383-a8e1-661926a66d45" ] }, { "ref": "store-health-sweep/5", "stepId": "write-health", "activatedAt": "2026-11-05T08:48:33.851Z", "completedAt": "2026-11-05T09:05:18.945Z", "ms": 1005094, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "b336cf37-752f-471e-b90b-71feec455389" ] }, { "ref": "store-health-sweep/6", "stepId": "write-health", "activatedAt": "2026-11-05T08:48:33.897Z", "completedAt": "2026-11-05T09:12:28.366Z", "ms": 1434469, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "29c867f7-8b1e-41f2-9496-13ea2dc3fd75" ] }, { "ref": "store-health-sweep/7", "stepId": "write-health", "activatedAt": "2026-11-05T08:48:33.939Z", "completedAt": "2026-11-05T09:19:44.954Z", "ms": 1871015, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "9ad4112c-1ef4-4f11-bc7a-526d72c1d110" ] }, { "ref": "store-health-sweep/3", "stepId": "finish", "activatedAt": "2026-11-05T09:32:11.284Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "bd82bc9e-2729-4369-9ceb-c406b598432c" ] }, { "ref": "store-health-sweep/8", "stepId": "load", "activatedAt": "2026-11-05T09:47:17.643Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T06:01:08.914Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T06:01:08.830Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 931904 }, "sessions": { "total": 17, "taskScoped": 15, "byGezel": { "darius": 1, "frida": 3, "kebba": 8, "reza": 1, "viorica": 4 }, "perTask": { "store-health-sweep/2": 4, "store-health-sweep/1": 4, "store-health-sweep/3": 3, "store-health-sweep/6": 1, "store-health-sweep/4": 1, "store-health-sweep/7": 1, "store-health-sweep/5": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7894736842105263, "maxMessages": 19, "maxContextFill": 0.5356903076171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 4, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5356903076171875 }, "fanout": { "hosts": 2, "childrenSpawned": 4, "childrenCompleted": 4, "childrenFailed": 0, "barrierHolds": 1, "barrierReleases": 1, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r1-saluki-cuda-complex-work-craftbook-author-fanout", "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "scenarioId": "craftbook-author-fanout", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/workspace/store-health-sweep/out/harbor-health.md", "artifactBytes": 745, "artifactSha256": "61af50c42d23729eb420d3c7796931bf793506554db16d61a4dc0e9cf9119c7c", "axisSource": "evals/src/craftbooks/authoring/author-fanout.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. Version1.2 really spawned four children; all completed and wrote their reports. Numbers match the source, but Harbor invents backlog growth and Linden invents a 30-day cadence. The grader overlooked that graph and passed on a fourth root invocation; version1.3 had only reached load. The working parent remained at finish.", "reviewBasis": "Read all final reports, raw stores, task/child snapshots, authored-document chronology, output writes and grader selection code. Compared earlier checkpoints and the false child claim against eventual real child completion; kept the fixed pass separate from version and lifecycle evidence.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/workspace/store-health-sweep/out/harbor-health.md", "store": { "slug": "harbor", "name": "Harbor Books", "city": "Rotterdam", "openIssues": 7, "lastAuditDays": 41 }, "bytes": 745, "sha256": "61af50c42d23729eb420d3c7796931bf793506554db16d61a4dc0e9cf9119c7c" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/workspace/store-health-sweep/out/linden-health.md", "store": { "slug": "linden", "name": "Linden Florists", "city": "Utrecht", "openIssues": 2, "lastAuditDays": 12 }, "bytes": 500, "sha256": "fc575fb5d265706a7cb8dd69283533bb1d7f64f9ffec367be79c3d19e8e942f7" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/workspace/store-health-sweep/out/northside-health.md", "store": { "slug": "northside", "name": "Northside Gym", "city": "Groningen", "openIssues": 13, "lastAuditDays": 96 }, "bytes": 501, "sha256": "cebc320b76ef63fbf72f0fd02dc7a8518fa8c30c3d07944ebf61209ce9c2b4b8" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/workspace/store-health-sweep/out/milo-health.md", "store": { "slug": "milo", "name": "Milo's Deli", "city": "Delft", "openIssues": 0, "lastAuditDays": 5 }, "bytes": 422, "sha256": "8bfc91cb971e568171bb41d9c5d88ac4188e04b9ce9865d6717a858b3a5bf489" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:04.598Z", "reviewId": "043", "cellId": "r1-saluki-cuda-complex-work-craftbook-author-fanout", "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "scenarioId": "craftbook-author-fanout", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-fanout", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "5ef82a30a2855a1c550469c93039fb7d17b0d6659df23db877caebe4d708cda0", "evidenceSha256": "e98867561f7cc70000472bf19a0cbf707251b23d179589736f6175e23eb50343", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/workspace/store-health-sweep/out/harbor-health.md", "bytes": 745, "sha256": "61af50c42d23729eb420d3c7796931bf793506554db16d61a4dc0e9cf9119c7c", "shownBytes": 745, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r1-saluki-cuda-complex-work-craftbook-author-fanout\\craftbook-author-fanout\\craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04\\workspace\\store-health-sweep\\data\\stores.json", "bytes": 517, "sha256": "8bf728bad631198134c68817a9d08eaf08804c45fe27af05c1d8433c7e573e22", "shownBytes": 517, "truncated": false } ], "scoreAxes": {}, "justification": "Only harbor-health.md is shown, and it matches stores.json (7 issues, 41 days, sensible advice). Its \"backlog building up\" wording is unsupported by the data. The craftbook, its fan-out design, and the other three outputs are not in the evidence, so the core deliverable is unverified.", "limitations": "The craftbook, tool-call trace, and the linden, northside and milo outputs were not provided. I can't confirm the book was authored before delegation, that it works off a growing list, or that all four files exist. The runner outcome is terminal but not interpreted.", "meanScore": null, "durationMs": 15460, "costUsd": 0.01375, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2659, "cache_read_input_tokens": 700, "output_tokens": 304, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2659, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 304, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2659, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2659 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 304, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2659, "webSearchRequests": 0, "costUSD": 0.01375, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2659, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-fanout / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.1 / 10** (band: ship-ready)\n**Outcome: success** (duration: 226m 24s; budget used: 85.8%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1209655`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-fanout\",\"bytes\":4047,\"score\":8,\"scoreMax\":8,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.8584280786563547`; `facts.toolUse.totalToolCalls: 217`; `facts.timing.timeToFirstArtifactMs: 1209655`; `facts.outcome.durationMs: 13583917` |\n| Behavior soundness (15%) | 7.5 / 10 | No red flags or missing roles were observed; two interventions receive the midpoint between the fixed low/high-intervention anchors. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 2` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 7.5 = **8.1**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15539 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10648 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30882 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.74 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 671221 + 94025 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 6s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 19 → 15 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 932s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 15 / 0.79 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 4 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 54% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 4 / 4 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 1 / 1 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 4 / 8 of 10 / 40 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1209655`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-fanout\",\"bytes\":4047,\"score\":8,\"scoreMax\":8,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.8584280786563547`\n- `facts.toolUse.totalToolCalls: 217`\n- `facts.timing.timeToFirstArtifactMs: 1209655`\n- `facts.outcome.durationMs: 13583917`\n\n### Behavior soundness - B_TWO_INTERVENTIONS\n\nNo red flags or missing roles were observed; two interventions receive the midpoint between the fixed low/high-intervention anchors.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 2`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1209655`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-fanout\",\"bytes\":4047,\"score\":8,\"scoreMax\":8,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.8584280786563547`; `facts.toolUse.totalToolCalls: 217`; `facts.timing.timeToFirstArtifactMs: 1209655`; `facts.outcome.durationMs: 13583917`\n- Behavior: No red flags or missing roles were observed; two interventions receive the midpoint between the fixed low/high-intervention anchors. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 2`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass,8.1/10** is preserved.\n\nNo declared axes; prose-only advisory review.\n\nNo declared axes. Version1.2 really spawned four children; all completed and wrote their reports. Numbers match the source, but Harbor invents backlog growth and Linden invents a 30-day cadence. The grader overlooked that graph and passed on a fourth root invocation; version1.3 had only reached load. The working parent remained at finish.\n\nRead all final reports, raw stores, task/child snapshots, authored-document chronology, output writes and grader selection code. Compared earlier checkpoints and the false child claim against eventual real child completion; kept the fixed pass separate from version and lifecycle evidence.\n\n## Positives\n\n- Version1.2.0 declares top-level spawn over the per-task store array and a spawnFanout activation step. It created four real children under task3; all four completed their write-health step, and the barrier released (continuity.fanout.childrenSpawned: 4; childrenCompleted: 4; barrierReleases: 1; saved child states). This is genuine per-item execution beyond merely claiming it in prose.\n- All four final files contain the correct store/city, open-issue count and audit age: Harbor7/41, Linden2/12, Northside13/96, Milo0/5. Different recommendations reflect those inputs (sniff.latest.score: 8/8; toolUse.byTool.write_file: 20; advisory source/output audit).\n- A repeated identical invoke at08:48:14 reused task3 rather than duplicating it. The run eventually corrected its invented step.fanout to the supported declaration (toolUse.byTool.invoke_craftbook: 5; craftbook_write: 8; recorded idempotency response and template snapshots).\n\n## Negatives\n\n- The grader selects an older completed root via findTaskForCraftbook, sees no children there, and falls back to counting matching roots. It remained at7/8 despite four real children, then passed when a fourth root/task8 was created at09:47:17. Version1.3 is still on load at capture, while the working version1.2 parent remains on finish. The pass combines latest template shape, old completed-root selection, root count and previously written outputs (outcome.success; sniff progression; continuity.steps; final states; authoring/helpers.ts and author-fanout.ts).\n- The initial version is sequential. Version1.1 submitted an unsupported steps[].fanout object that was accepted but stripped; its fanout.md claimed four completed children while list_task_children returned none. Only version1.2 later established real fanout. Preserve both the earlier unsupported claim and eventual recovery (toolUse.byTool.craftbook_write: 8; list_task_children: 5;07:49 structure audit and final child states).\n- Harbor infers that issues arrive faster than they clear from one snapshot; no rate history exists. Linden calls the audit cadence30 days, although the author rubric uses <30 only as part of a healthy-condition rule and the source has no cadence. Northside also infers issue complexity from count. Final numeric facts are accurate, but these added interpretations are unsupported (sniff.latest artifact evidence; final Markdown versus source; toolUse.byTool.write_file: 20).\n- The grounding guard rejected the Harbor child’s correct7/41 values although its interpolated task prompt contains them, then accepted a rewrite adding the unsupported growth claim. Later it rejected verifier file-size facts despite preceding stat calls, and accepted citation-style rewording. Audit evidence propagation and claim matching rather than treating prose style as proof (failed write events08:54:30 and09:27:32; toolUse.byTool.stat: 9; saved child prompt and successful rewrites).\n- The226m24s run consumed217 tool calls,18 craftbook reads,8 writes and5 invocations across17 sessions. Four mid-turn compactions occurred; no context overflow or CUDA OOM was logged. Most overhead concerns authoring/lineage and repeated validation rather than loading the model (outcome.durationMs; toolUse; continuity.sessions.total and compaction.midTurn).\n\n## Tactical experiment — not applied\n\nIn AUTHORING_WORKER_ABOUT in evals/src/craftbooks/authoring/helpers.ts, add a generic authoring instruction to read the canonical schema, save the document, and read it back to confirm its structural fields before invoking. State that a count snapshot supports a workload assessment, not a trend; proposed thresholds must be labeled as policy choices. Test this on per-item reports and an unrelated parameterized export. No model-specific scenario names or expected output strings.\n\n## Strategic experiment — not applied\n\nIn packages/core/src/craftbook-doc.ts authoring parsing, reject or explain discarded unknown structural keys (for example steps[].fanout) while preserving tolerant catalog reads. In evals/src/craftbooks/authoring/author-fanout.ts, select a consistent template-version/root/child lineage and count that host’s children, not repeated invocations; record output writes and terminal state separately. In the service factual-grounding pipeline, carry trusted interpolated task values and stat receipts into claim evidence, and do not treat citation-like qualifiers as support for invented trends. Validate revised recipes, actual fanout children and unrelated document/metadata tasks without changing this frozen campaign.\n\n## Performance and GPU fit\n\nElapsed 226.40min; recorded output 94025tokens, mean decode8.74tok/s, prefill320.92tok/s. Peak VRAM10648/12227MiB, process-tree RSS15539MiB, whole-system RAM30882/31865MiB. One native0.1.57 CUDA launch,65536-token context, one slot,f16KV,reasoning4096; dense FFN from39/64layers GPU-resident and25 on CPU. Shared GPU memory/paging unmeasured. No explicit CUDA OOM, context overflow or unexpected native exit observed. Performance remains separate from fixed capability and advisory quality.\n\n## Generalization and recommended next experiment\n\nGate-script accepted a task label without a successful gate, author-params stopped before both tasks completed, and this trial mixed roots/versions while missing a real child graph. These support a general receipt-and-lineage invariant across workflows. Unlike the gate-script trial, real per-item execution is established here by four completed child steps. The next experiment should bind graders to one executable lineage and compare fanout with parameterized and reviewer workflows, expecting fewer stale repair nudges without crediting unsupported completion claims.\n\n## Last three same-scenario trials\n\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556/facts.json): fail,fixed3.5,12.03min,99.32tok/s. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: store health sweep; missing signals stayed unchanged: no reusable authored craftbook exists yet — author the store health sweep as a craftbook template, not as ad-hoc chat work, out/harbor-health.md is 0 bytes, need ≥ 60, out/linden-health.md is 0 bytes, need ≥ 60, out/northside-health.md i\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c/facts.json): fail,fixed4.5,8.71min,109.22tok/s. repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: store health sweep; missing signals stayed unchanged: craftbook \"store-health-sweep\" declares no per-item fanout over the store list — the recipe must derive its per-store work from the store list itself (one item, one run) rather than repeating steps once per store, craftbook \"store-health\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T19-50-46-962Z-8721/facts.json): fail,fixed3,28.00min,98.25tok/s. chat stalled for 305s — daemon reachable but issuing no model turns (soft digest stuck at 82f1867183be8622) (re-engage nudge was sent and ignored)\n\nThese are historical context except where the same frozen campaign identity applies. No causal native-version improvement is inferred from other models/releases.\n\n## Strategic note\n\nReal fanout was eventually achieved, but the score transition was driven by a mixed-lineage count and another invocation. Separate the four completed child jobs from the unexecuted newest template and unfinished parent closure. No source, gate, prompt, tuning or model changes were applied and no failed capability cell was rerun.\n\n## Raw evidence\n\nSee facts.json,score.json,advisory-artifact-review.json,advisory-workflow-audit.json and historical-comparison.json.\n" }, { "cellId": "r2-saluki-cuda-core-tictactoe", "baselineCellId": "r2-saluki-cuda-core-tictactoe", "suite": "core", "scenario": "tictactoe", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T09:50:34.103Z", "finishedAt": "2026-11-05T09:56:14.567Z", "durationMs": 340464, "success": true, "reason": "tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 330830 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 3947, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": 2, "runtimeFailed": 0 } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 3947 } ], "adjustments": [ { "ruleId": "Q_HTML_REALISM_CAP", "summary": "Primary game HTML is 3947 bytes, below the 4096-byte realism floor; quality is capped at 5." } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.021515433094272965 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 330830 }, { "source": "facts", "path": "outcome.durationMs", "value": 340464 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" }, "facts": { "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 56, "peakRssMb": 10847, "peakCpuPercent": 1211.1, "firstAtMs": 0, "lastAtMs": 323981 }, "gpu": { "available": true, "sampleCount": 56, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 56, "peakUsedMb": 27802, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12045, "totalOutputTokens": 2404, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 14730, "todayTokensCached": 13782, "todayTokensOut": 504, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 14730, "totalTokensCached": 13782, "totalTokensOut": 504, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7430, "cachedInputTokens": 6891, "outputTokens": 400, "cost": 0, "durationMs": 49865, "outputTokensPerSec": 8.451624658561778, "at": "2026-11-05T09:53:12.551Z" }, "lastUpdated": "2026-11-05T09:53:12.551Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T09:56:09.007Z" } }, "source": "engine-log" }, "derived": { "durationMs": 328707, "meanTokensPerSec": 8.52, "genTokensPerSec": 8.52, "genTokens": 2404, "promptTokensPerSec": 473.04, "peakRssMb": 10847, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 14730, "output": 504 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tic-tac-toe-game/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-05T09:52:08.495Z", "completedAt": "2026-11-05T09:56:05.603Z", "ms": 237108, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "aae54eaf-c8e6-44ad-aaf4-cd13f9988464" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T09:50:39.838Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T09:50:39.737Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 237108 }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "aicha": 1, "ambrose": 1 }, "perTask": { "tic-tac-toe-game/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 5, "maxContextFill": 0.10174560546875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.10174560546875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "durationMs": 340464, "timeoutMs": 15824176, "budgetUsedFraction": 0.021515433094272965 }, "timing": { "startedAt": "2026-11-05T09:50:34.103Z", "finishedAt": "2026-11-05T09:56:14.567Z", "timeToFirstArtifactMs": 330830, "timeToLastArtifactWriteMs": 330867, "timeToFirstTokenMs": 30175, "firstTurnTtftMs": 19743, "timeToFirstToolCallMs": 94448 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "start_work": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/tic-tac-toe-game/index.html", "finalBytes": 3949, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 331187, "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 3947, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null } ], "latest": { "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 3947, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": 2, "runtimeFailed": 0 } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T09:50:34.106Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-tictactoe-3XmRHA", "[2026-11-05T09:50:34.106Z] [trial] id=tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc scenario=tictactoe model=underdog-saluki-27b-iq2-mix", "[2026-11-05T09:50:34.106Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "[2026-11-05T09:50:34.106Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T09:50:34.106Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T09:50:34.108Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T09:50:34.112Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T09:50:34.117Z] [trial] throughput-scaled ceiling: 120m → 264m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T09:50:34.117Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-tictactoe-3XmRHA", "[2026-11-05T09:50:39.771Z] [trial] daemon spawned pid=52448 port=https://127.0.0.1:63338", "[2026-11-05T09:50:41.128Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T09:50:41.145Z] [trial] meester=ambrose", "[2026-11-05T09:50:41.298Z] [trial] sent prompt to meester (621 chars)", "[2026-11-05T09:50:41.355Z] [poll] initial digests hard=41b17acbf0030316 soft=b12252683b7d54bb", "[2026-11-05T09:52:09.605Z] [poll] hard-progress: 41b17acbf0030316 → e9574e644de02b54 after 88s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=176,stream=691) sniff=none)", "[2026-11-05T09:53:14.230Z] [poll] hard-progress: e9574e644de02b54 → d5eff6c7325fbb1d after 65s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=1,slot=348,stream=1187) sniff=none)", "[2026-11-05T09:56:08.465Z] [poll] terminal: success=true reason=tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "[2026-11-05T09:56:09.054Z] [capture] dumped 2 session(s)", "[2026-11-05T09:56:09.058Z] [capture] snapshotted telemetry for 2 session(s)", "[2026-11-05T09:56:09.118Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T09:56:14.565Z] [trial] daemon shut down", "[2026-11-05T09:56:14.575Z] [trial] PASS duration=340464ms reason=tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))" ] }, "metrics": { "process": { "sampleCount": 56, "peakRssMb": 10847, "peakCpuPercent": 1211.1, "firstAtMs": 0, "lastAtMs": 323981 }, "gpu": { "available": true, "sampleCount": 56, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 56, "peakUsedMb": 27802, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12045, "totalOutputTokens": 2404, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 14730, "todayTokensCached": 13782, "todayTokensOut": 504, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 14730, "totalTokensCached": 13782, "totalTokensOut": 504, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7430, "cachedInputTokens": 6891, "outputTokens": 400, "cost": 0, "durationMs": 49865, "outputTokensPerSec": 8.451624658561778, "at": "2026-11-05T09:53:12.551Z" }, "lastUpdated": "2026-11-05T09:53:12.551Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T09:56:09.007Z" } }, "source": "engine-log" }, "derived": { "durationMs": 328707, "meanTokensPerSec": 8.52, "genTokensPerSec": 8.52, "genTokens": 2404, "promptTokensPerSec": 473.04, "peakRssMb": 10847, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 14730, "output": 504 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T09:50:34.103Z", "finishedAt": "2026-11-05T09:56:14.567Z", "timeToFirstArtifactMs": 330830, "timeToLastArtifactWriteMs": 330867, "timeToFirstTokenMs": 30175, "firstTurnTtftMs": 19743, "timeToFirstToolCallMs": 94448 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tic-tac-toe-game/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-05T09:52:08.495Z", "completedAt": "2026-11-05T09:56:05.603Z", "ms": 237108, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "aae54eaf-c8e6-44ad-aaf4-cd13f9988464" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T09:50:39.838Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T09:50:39.737Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 237108 }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "aicha": 1, "ambrose": 1 }, "perTask": { "tic-tac-toe-game/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 5, "maxContextFill": 0.10174560546875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.10174560546875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-tictactoe", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "scenarioId": "tictactoe", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/workspace/tic-tac-toe-game/index.html", "artifactBytes": 3949, "artifactSha256": "40b6117623d35cd7f9ce649498c8eb1a3b5aa9854c7de81a58e4605da4e4658c", "axisSource": "evals/src/llm-judge.ts DEFAULT_HTML_AXES", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 6, "functionalCompleteness": 7, "codeQuality": 7, "polish": 4 }, "meanScore": 6, "justification": "Clean, readable board; both players can win, draws resolve, occupied/end-state moves are blocked, and reset works. Keyboard activation and320px layout pass. Code is concise. Static cell names omit marks and status is not announced live; styling and feedback remain basic beyond hover and a winning-line highlight.", "reviewBasis": "Read the full HTML within20KiB; inspected captured desktop and320px mobile screenshots. Ran independent headless checks of both winners, draw, reset, occupied/post-win guards, keyboard activation and horizontal overflow. No page errors; advisory only.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/workspace/tic-tac-toe-game/index.html", "bytes": 3949, "sha256": "40b6117623d35cd7f9ce649498c8eb1a3b5aa9854c7de81a58e4605da4e4658c" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:13.063Z", "reviewId": "044", "cellId": "r2-saluki-cuda-core-tictactoe", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "scenarioId": "tictactoe", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "baselineCellId": "r2-saluki-cuda-core-tictactoe", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "70dc32e04966bdf716442847eeb7deeede110850d66ffe501af42482330c5971", "evidenceSha256": "b874e9a4bee1b19edf9f3dd183d0cc334eac197345a5cd34a0f1815ada355af6", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/workspace/tic-tac-toe-game/index.html", "bytes": 3949, "sha256": "40b6117623d35cd7f9ce649498c8eb1a3b5aa9854c7de81a58e4605da4e4658c", "shownBytes": 3949, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 6, "functionalCompleteness": 9, "codeQuality": 8, "polish": 5 }, "justification": "Nine static .cell buttons with data-cell 0–8 are wired via querySelectorAll. Turns alternate, wins and draws are detected, and the winner is shown. The code is clean, with a WINS table and a reset function. The look is tidy: coloured X and O, a green win highlight, a styled button. Cells are small at 80px and there is no scoreboard or animation.", "limitations": "Source-only review; nothing was rendered or run. The file sits at workspace/tic-tac-toe-game/index.html, which I read as the new project folder, not literally workspace/index.html. Layout and behaviour are inferred from the code.", "meanScore": 7, "durationMs": 16298, "costUsd": 0.0207496, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3665, "cache_read_input_tokens": 856, "output_tokens": 600, "output_tokens_details": { "thinking_tokens": 244 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3665, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 600, "cache_read_input_tokens": 856, "cache_creation_input_tokens": 3665, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3665 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 600, "cacheReadInputTokens": 856, "cacheCreationInputTokens": 3665, "webSearchRequests": 0, "costUSD": 0.0207496, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 244, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3665, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - tictactoe / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.8 / 10** (band: ship-ready)\n**Outcome: success** (duration: 5m 40s; budget used: 2.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 5 / 10 | The scenario gate is complete. Primary game HTML is 3947 bytes, below the 4096-byte realism floor; quality is capped at 5. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 330830`; `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":3947,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`; `facts.sniff.latest.bytes: 3947` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.021515433094272965`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 330830`; `facts.outcome.durationMs: 340464` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **8.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10847 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27802 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.52 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 12045 + 2404 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 237s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 10% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- Adjustment Q_HTML_REALISM_CAP: Primary game HTML is 3947 bytes, below the 4096-byte realism floor; quality is capped at 5.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 330830`\n- `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":3947,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`\n- `facts.sniff.latest.bytes: 3947`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.021515433094272965`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 330830`\n- `facts.outcome.durationMs: 340464`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 330830`; `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":3947,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`; `facts.sniff.latest.bytes: 3947`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.021515433094272965`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 330830`; `facts.outcome.durationMs: 340464`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass,8.8/10** is preserved.\n\n| Axis | Advisory score |\n|---|---:|\n| visualQuality | 6/10 |\n| functionalCompleteness | 7/10 |\n| codeQuality | 7/10 |\n| polish | 4/10 |\n| Mean |6.0/10|\n\nClean, readable board; both players can win, draws resolve, occupied/end-state moves are blocked, and reset works. Keyboard activation and320px layout pass. Code is concise. Static cell names omit marks and status is not announced live; styling and feedback remain basic beyond hover and a winning-line highlight.\n\nRead the full HTML within20KiB; inspected captured desktop and320px mobile screenshots. Ran independent headless checks of both winners, draw, reset, occupied/post-win guards, keyboard activation and horizontal overflow. No page errors; advisory only.\n\n## Positives\n\n- One start_work and one write_file produced a playable self-contained HTML game in5m40s, with no auto-answer or red-flag event (toolUse.totalToolCalls: 2; byTool; outcome.durationMs: 340464; autoAnswer.total: 0; toolUse.redFlags: []).\n- The standard sniff/runtime gate passed. Independent advisory browser checks also passed both X/O wins, a draw, occupied-cell and post-win guards, reset, and keyboard button activation, with zero page errors (outcome.success; sniff.latest; advisory-runtime-check.json).\n- The centered260px board fits a320px viewport with no horizontal overflow. Turn status, contrasting player colors and a winning-line highlight make state visible; the source is a short clear state machine (artifacts.htmlFiles; desktop/mobile screenshots; advisory runtime evidence).\n\n## Negatives\n\n- The static aria-label remains Cell1 after it contains X, and the status has neither aria-live nor role=status. Keyboard input works, but screen-reader users do not receive a complete move/result announcement (artifacts.htmlFiles final source; advisory-runtime-check.accessibility).\n- Visual design is clean but minimal: generic styling, no special transition beyond hover and winning-cell color. This is a usable small game rather than a polished product surface (artifacts.htmlFiles; inspected screenshots; separate advisory polish4/10).\n\n## Tactical experiment — not applied\n\nIn the generic web UI authoring guidance used by the writer/developer role, require interactive state changes to update accessible names and announce outcomes with a polite live region. Place the guidance in the applicable ../gilde/data/gezel-templates role about.md after inspecting the template; keep it independent of games or selectors. Verify on this output and an unrelated interactive form. No sampling change is supported by this clean two-call run.\n\n## Strategic experiment — not applied\n\nExtend reusable browser-artifact diagnostics in packages/mcp/src/validate.ts to surface missing dynamic accessible names and live status announcements as advisory findings. Validate with an unrelated form/dashboard. Keep the anchored tictactoe scenario and fixed gates unchanged; fix generic authoring and validation rather than adding scenario-specific recipes.\n\n## Performance and GPU fit\n\nElapsed 5.67min; recorded output 2404tokens, mean decode8.52tok/s, prefill473.04tok/s. Peak VRAM10626/12227MiB, process-tree RSS10847MiB, whole-system RAM27802/31865MiB. One native0.1.57 CUDA launch,65536-token context, one slot,f16KV,reasoning4096; dense FFN from39/64layers GPU-resident and25 on CPU. Shared GPU memory/paging unmeasured. No explicit CUDA OOM, context overflow or unexpected native exit observed. Performance remains separate from fixed capability and advisory quality.\n\n## Generalization and recommended next experiment\n\nRepetition1 also passed8.8 with two calls, taking479457ms and3600 generated tokens at8.68tok/s. Repetition2 took340464ms and2404 tokens at8.52tok/s: less generated work, not a faster engine. Both advisory means are6.0. Contrast the217-call fanout run: simple complete-file delivery is reliable here, while workflow authoring and lifecycle validation dominate difficult-task cost. Two trials are not a three-trial rate or evidence of overall diverse competence. Next experiment: generic accessible state guidance on this game and an unrelated form, measuring announced state and regression of core behavior.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/facts.json): pass,fixed8.8,7.99min,8.68tok/s. tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/facts.json): pass,fixed10,0.82min,137.58tok/s. this-work-e/workspace/index.html passed sniff; runtime layer unavailable\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-03-125Z-ae1q/facts.json): pass,fixed10,0.86min,140.25tok/s. tic-tac-toe-game/workspace/index.html passed sniff; runtime layer unavailable\n\nThese are historical context except where the same frozen campaign identity applies. No causal native-version improvement is inferred from other models/releases.\n\n## Strategic note\n\nThe artifact meets its two-player brief and core interaction checks. Accessibility and visual polish remain separate advisory weaknesses; a working game does not establish competence across complex workflows. No source, gate, prompt, tuning or model changes were applied and no failed capability cell was rerun.\n\n## Raw evidence\n\nSee facts.json,score.json,advisory-artifact-review.json,advisory-workflow-audit.json and historical-comparison.json.\n" }, { "cellId": "r2-saluki-cuda-core-petshop", "baselineCellId": "r2-saluki-cuda-core-petshop", "suite": "core", "scenario": "petshop", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T09:59:21.726Z", "finishedAt": "2026-11-05T11:02:16.358Z", "durationMs": 3774632, "success": true, "reason": "pet-shop-site/workspace/index.html passed sniff (signals: pet-vocab, store-vocab, structured-page, working-image, image-asset) + runtime (1 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "requiredRasterMinBytes": 51200 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 986955 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "pet-shop-site/workspace/index.html", "bytes": 19497, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": 1, "runtimeFailed": 0 } }, { "source": "facts", "path": "artifacts.imageFiles", "value": [ { "path": "workspace/pet-shop-site/assets/logo.png", "bytes": 1238904 }, { "path": "artifacts/pet-shop-site/generated/image-2026-11-05T11-01-59-559Z-1254255121.png", "bytes": 1238904 }, { "path": "artifacts/pet-shop-site/sessions/2026-11-05_100623_new-session/tool-0-img-0.png", "bytes": 1238904 } ] } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2385357695718248 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 42 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 986955 }, { "source": "facts", "path": "outcome.durationMs", "value": 3774632 } ] }, "behavior": { "score": 0, "ruleId": "B_RED_FLAG_OR_MISSING_ROLE", "summary": "At least one red-flag behavior fired or an expected role was missing.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [ "image-generator" ] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 7, "band": "needs-tuning" }, "facts": { "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 630, "peakRssMb": 15467, "peakCpuPercent": 1240.6, "firstAtMs": 1, "lastAtMs": 3755174 }, "gpu": { "available": true, "sampleCount": 630, "peakUtilPercent": 100, "peakMemUsedMb": 10654, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 630, "peakUsedMb": 31242, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 135038, "totalOutputTokens": 29522, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 70552, "todayTokensCached": 40907, "todayTokensOut": 5226, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 70552, "totalTokensCached": 40907, "totalTokensOut": 5226, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10649, "outputTokens": 163, "cost": 0, "durationMs": 999913, "outputTokensPerSec": 9.169320675595547, "at": "2026-11-05T10:49:36.704Z" }, "lastUpdated": "2026-11-05T10:49:36.704Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:02:05.975Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3757787, "meanTokensPerSec": 8.97, "genTokensPerSec": 8.97, "genTokens": 29522, "promptTokensPerSec": 382.97, "peakRssMb": 15467, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 70552, "output": 5226 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "pet-shop-site/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-05T10:01:44.412Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "dc2a260d-34d5-4828-8acf-9385eccd5acb" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T09:59:27.866Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T09:59:27.764Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 6, "taskScoped": 1, "byGezel": { "desanka": 1, "kseniya": 1, "nisa": 1, "orion": 2, "vitaly": 1 }, "perTask": { "pet-shop-site/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 7, "maxContextFill": 0.2869873046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2869873046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 6 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "pet-shop-site/workspace/index.html passed sniff (signals: pet-vocab, store-vocab, structured-page, working-image, image-asset) + runtime (1 assertion(s))", "durationMs": 3774632, "timeoutMs": 15824176, "budgetUsedFraction": 0.2385357695718248 }, "timing": { "startedAt": "2026-11-05T09:59:21.726Z", "finishedAt": "2026-11-05T11:02:16.358Z", "timeToFirstArtifactMs": 986955, "timeToLastArtifactWriteMs": 3692332, "timeToFirstTokenMs": 28776, "firstTurnTtftMs": 18488, "timeToFirstToolCallMs": 142747 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer", "Illustrator", "Builder" ], "missingExpectedRoles": [ "image-generator" ] }, "toolUse": { "totalToolCalls": 42, "byTool": { "start_work": 1, "search_sessions": 7, "search_history": 3, "search": 2, "list_gezels": 1, "delegate_image_generator": 2, "ensure_gezel": 2, "message_gezel": 3, "read_task_notes": 2, "write_file": 14, "read_file": 2, "replace_in_file": 1, "generate_image": 1, "delegate_builder": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/pet-shop-site/index.html", "finalBytes": 19511, "growth": [] } ], "imageFiles": [ { "path": "workspace/pet-shop-site/assets/logo.png", "bytes": 1238904 }, { "path": "artifacts/pet-shop-site/generated/image-2026-11-05T11-01-59-559Z-1254255121.png", "bytes": 1238904 }, { "path": "artifacts/pet-shop-site/sessions/2026-11-05_100623_new-session/tool-0-img-0.png", "bytes": 1238904 } ], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 990967, "filePath": "pet-shop-site/workspace/index.html", "bytes": 12082, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 2202329, "filePath": "pet-shop-site/workspace/index.html", "bytes": 12105, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 2601641, "filePath": "pet-shop-site/workspace/index.html", "bytes": 9234, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 3696913, "filePath": "pet-shop-site/workspace/index.html", "bytes": 19497, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 3762579, "filePath": "pet-shop-site/workspace/index.html", "bytes": 19497, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null } ], "latest": { "filePath": "pet-shop-site/workspace/index.html", "bytes": 19497, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": 1, "runtimeFailed": 0 } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T09:59:21.729Z] [trial] id=petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl scenario=petshop model=underdog-saluki-27b-iq2-mix", "[2026-11-05T09:59:21.729Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "[2026-11-05T09:59:21.729Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T09:59:21.729Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T09:59:21.729Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-petshop-LfENwh", "[2026-11-05T09:59:21.733Z] [trial] sd-server=packages\\app\\native-bin\\win32-x64\\gezel-sd-server.exe", "[2026-11-05T09:59:21.732Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T09:59:21.736Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T09:59:21.738Z] [trial] sd-cpp source=\\.gezel-eval-cache\\engines\\sd-cpp\\models\\sdxl-lightning-4step (read-only)", "[2026-11-05T09:59:21.744Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-petshop-LfENwh", "[2026-11-05T09:59:21.747Z] [trial] throughput-scaled ceiling: 120m → 264m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T09:59:21.747Z] [trial] cloned sd-cpp/sdxl-lightning-4step from \\.gezel-eval-cache\\engines\\sd-cpp\\models\\sdxl-lightning-4step into \\AppData\\Local\\Temp\\gezel-eval-petshop-LfENwh", "[2026-11-05T09:59:27.855Z] [trial] daemon spawned pid=1864 port=https://127.0.0.1:62957", "[2026-11-05T09:59:29.111Z] [trial] provider=llama-cpp imageProvider=sd-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T09:59:29.129Z] [trial] meester=orion", "[2026-11-05T09:59:29.295Z] [trial] sent prompt to meester (795 chars)", "[2026-11-05T09:59:29.350Z] [poll] initial digests hard=474952f5718bdaf3 soft=2ecfca3936278735", "[2026-11-05T10:01:45.480Z] [poll] hard-progress: 474952f5718bdaf3 → fc86e9723053e8d0 after 136s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=337,stream=1316) sniff=none)", "[2026-11-05T10:02:50.842Z] [poll] hard-progress: fc86e9723053e8d0 → cdfd111857ebb514 after 65s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=1,slot=523,stream=1866) sniff=none)", "[2026-11-05T10:04:14.432Z] [poll] hard-progress: cdfd111857ebb514 → 1c709cebffb52f50 after 84s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=3,slot=593,stream=2049) sniff=none)", "[2026-11-05T10:05:07.610Z] [poll] hard-progress: 1c709cebffb52f50 → 7fe455ce98262aa0 after 53s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=5,slot=739,stream=2482) sniff=none)", "[2026-11-05T10:06:25.103Z] [poll] hard-progress: 7fe455ce98262aa0 → f9e254da68a54d74 after 77s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=3,tools=7,slot=923,stream=3484) sniff=none)", "[2026-11-05T10:15:52.907Z] [poll] hard-progress: f9e254da68a54d74 → cba7fccd8621bf48 after 568s (workspaceProjects=3 sessions=5 daemon[src=service-telemetry](turns=3,tools=8,slot=2525,stream=11631) sniff=3/12082B)", "[2026-11-05T10:16:34.372Z] [poll] hard-progress: cba7fccd8621bf48 → 042d9bc45617e6d2 after 41s (workspaceProjects=3 sessions=5 daemon[src=service-telemetry](turns=5,tools=10,slot=2610,stream=11873) sniff=3/12082B)", "[2026-11-05T10:17:04.207Z] [poll] hard-progress: 042d9bc45617e6d2 → d3acb52ef7e60f61 after 30s (workspaceProjects=3 sessions=5 daemon[src=service-telemetry](turns=5,tools=12,slot=2684,stream=12093) sniff=3/12082B)" ] }, "metrics": { "process": { "sampleCount": 630, "peakRssMb": 15467, "peakCpuPercent": 1240.6, "firstAtMs": 1, "lastAtMs": 3755174 }, "gpu": { "available": true, "sampleCount": 630, "peakUtilPercent": 100, "peakMemUsedMb": 10654, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 630, "peakUsedMb": 31242, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 135038, "totalOutputTokens": 29522, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 70552, "todayTokensCached": 40907, "todayTokensOut": 5226, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 70552, "totalTokensCached": 40907, "totalTokensOut": 5226, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10649, "outputTokens": 163, "cost": 0, "durationMs": 999913, "outputTokensPerSec": 9.169320675595547, "at": "2026-11-05T10:49:36.704Z" }, "lastUpdated": "2026-11-05T10:49:36.704Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:02:05.975Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3757787, "meanTokensPerSec": 8.97, "genTokensPerSec": 8.97, "genTokens": 29522, "promptTokensPerSec": 382.97, "peakRssMb": 15467, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 70552, "output": 5226 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T09:59:21.726Z", "finishedAt": "2026-11-05T11:02:16.358Z", "timeToFirstArtifactMs": 986955, "timeToLastArtifactWriteMs": 3692332, "timeToFirstTokenMs": 28776, "firstTurnTtftMs": 18488, "timeToFirstToolCallMs": 142747 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "pet-shop-site/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-05T10:01:44.412Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "dc2a260d-34d5-4828-8acf-9385eccd5acb" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T09:59:27.866Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T09:59:27.764Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 6, "taskScoped": 1, "byGezel": { "desanka": 1, "kseniya": 1, "nisa": 1, "orion": 2, "vitaly": 1 }, "perTask": { "pet-shop-site/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 7, "maxContextFill": 0.2869873046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2869873046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 6 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-petshop", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "scenarioId": "petshop", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/workspace/pet-shop-site/index.html", "artifactBytes": 19511, "artifactSha256": "b27ead1fed4c9cae18dd67eb65f60c5162a9ebf79ca4633f9a1be90ff29804d7", "axisSource": "evals/src/llm-judge.ts DEFAULT_HTML_AXES", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 5, "codeQuality": 4, "polish": 4 }, "meanScore": 4.5, "justification": "Styled storefront and real generated logo meet the main brief; cart add/remove totals work. Logo lettering is gibberish. Mobile navigation overflows and the 360px cart traps a 320px screen without a close button or Escape. The contact form promises a reply but sends nothing. Desktop presentation is coherent, with limited polish.", "reviewBasis": "Read all 19,511 HTML bytes, inspected the 1024px logo and desktop/mobile screenshots, and checked cart arithmetic, required-field validation, submission behavior and mobile navigation. Browser checks are advisory and do not change the fixed pass.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/workspace/pet-shop-site/index.html", "bytes": 19511, "sha256": "b27ead1fed4c9cae18dd67eb65f60c5162a9ebf79ca4633f9a1be90ff29804d7" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/workspace/pet-shop-site/assets/logo.png", "bytes": 1238904, "sha256": "14ad73a7804293a0bcc2d5f3d19db8b2209eb3e22b5d4d9e05dd93227845e955", "inspection": "Viewed original 1024x1024 raster; generated nonsense lettering." } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:16.168Z", "reviewId": "045", "cellId": "r2-saluki-cuda-core-petshop", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "scenarioId": "petshop", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "baselineCellId": "r2-saluki-cuda-core-petshop", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "d53cfb34208c625decc214b58754396426eecc278a9e85fa48cf1d593f19ce12", "evidenceSha256": "01d556ed55171a63419b247617006cf3934c8c2cf9c76f73d26e4af85d1903fb", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/workspace/pet-shop-site/index.html", "bytes": 19511, "sha256": "b27ead1fed4c9cae18dd67eb65f60c5162a9ebf79ca4633f9a1be90ff29804d7", "shownBytes": 19511, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 6, "functionalCompleteness": 6, "codeQuality": 7, "polish": 5 }, "justification": "index.html wires the logo exactly as \"Pet. It has a sticky header, hero, six category cards, services, about, a validated contact form, and a working cart drawer. The look is clean and warm, but text-only cards have no imagery. The code is tidy with CSS variables. Minor flaws: an unused .tag class and a raw \"&\" in the HTML.", "limitations": "Only index.html was shown. assets/logo.png was not in the evidence, so I can't confirm it exists or was generated with generate_image. I did not render the page or check the one-workspace requirement. Scores come from source reading only.", "meanScore": 6, "durationMs": 16321, "costUsd": 0.0478096, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 10470, "cache_read_input_tokens": 856, "output_tokens": 584, "output_tokens_details": { "thinking_tokens": 222 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 10470, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 584, "cache_read_input_tokens": 856, "cache_creation_input_tokens": 10470, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 10470 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 584, "cacheReadInputTokens": 856, "cacheCreationInputTokens": 10470, "webSearchRequests": 0, "costUSD": 0.0478096, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 222, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 10470, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - petshop / underdog-saluki-27b-iq2-mix\n\n**Composite: 7.0 / 10** (band: needs-tuning)\n**Outcome: success** (duration: 62m 55s; budget used: 23.9%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 986955`; `facts.sniff.latest: {\"filePath\":\"pet-shop-site/workspace/index.html\",\"bytes\":19497,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fail...`; `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-site/assets/logo.png\",\"bytes\":1238904},{\"path\":\"artifacts/pet-shop-site/generated/image-2026-11-05T11-01-59-559Z-1254255121.png\",\"bytes\":1238904},{\"...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2385357695718248`; `facts.toolUse.totalToolCalls: 42`; `facts.timing.timeToFirstArtifactMs: 986955`; `facts.outcome.durationMs: 3774632` |\n| Behavior soundness (15%) | 0 / 10 | At least one red-flag behavior fired or an expected role was missing. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: [\"image-generator\"]`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 0 = **7.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15467 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10654 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31242 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.97 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 135038 + 29522 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 29% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 6 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 986955`\n- `facts.sniff.latest: {\"filePath\":\"pet-shop-site/workspace/index.html\",\"bytes\":19497,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fail...`\n- `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-site/assets/logo.png\",\"bytes\":1238904},{\"path\":\"artifacts/pet-shop-site/generated/image-2026-11-05T11-01-59-559Z-1254255121.png\",\"bytes\":1238904},{\"...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.2385357695718248`\n- `facts.toolUse.totalToolCalls: 42`\n- `facts.timing.timeToFirstArtifactMs: 986955`\n- `facts.outcome.durationMs: 3774632`\n\n### Behavior soundness - B_RED_FLAG_OR_MISSING_ROLE\n\nAt least one red-flag behavior fired or an expected role was missing.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: [\"image-generator\"]`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 986955`; `facts.sniff.latest: {\"filePath\":\"pet-shop-site/workspace/index.html\",\"bytes\":19497,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fail...`; `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-site/assets/logo.png\",\"bytes\":1238904},{\"path\":\"artifacts/pet-shop-site/generated/image-2026-11-05T11-01-59-559Z-1254255121.png\",\"bytes\":1238904},{\"...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2385357695718248`; `facts.toolUse.totalToolCalls: 42`; `facts.timing.timeToFirstArtifactMs: 986955`; `facts.outcome.durationMs: 3774632`\n- Behavior: At least one red-flag behavior fired or an expected role was missing. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: [\"image-generator\"]`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 7/10** is preserved.\n\n| Axis | Advisory score |\n|---|---:|\n| visualQuality | 5/10 |\n| functionalCompleteness | 5/10 |\n| codeQuality | 4/10 |\n| polish | 4/10 |\n| Mean | 4.5/10 |\n\nStyled storefront and real generated logo meet the main brief; cart add/remove totals work. Logo lettering is gibberish. Mobile navigation overflows and the 360px cart traps a 320px screen without a close button or Escape. The contact form promises a reply but sends nothing. Desktop presentation is coherent, with limited polish.\n\nRead all 19,511 HTML bytes, inspected the 1024px logo and desktop/mobile screenshots, and checked cart arithmetic, required-field validation, submission behavior and mobile navigation. Browser checks are advisory and do not change the fixed pass.\n\n## Positives\n\n- The page and a real 1024×1024 generated PNG satisfy the main delivery brief. The final source includes its CSS and renders coherently; unlike repetition 1, it does not depend on a missing stylesheet (outcome.success; sniff.latest 5/5; artifacts.htmlFiles[0].finalBytes: 19511; artifacts.imageFiles; inspected screenshots).\n- The Illustrator actually called generate_image successfully once, using SDXL Lightning at four steps in about 19.7s. The runtime unloaded idle Saluki, ran SDXL, then reloaded Saluki and restored its session checkpoint (toolUse.byTool.generate_image: 1; recording/chat-events.jsonl; daemon.log 11:01:41–11:02:14; result.engineContext.launches: 2).\n- Cart quantity and total calculations passed: two Dog items plus one Cat item total $139.97; removing Dog leaves $39.99. Four empty-field errors appear, and no page error was observed (artifacts.htmlFiles; advisory-runtime-check.json).\n\n## Negatives\n\n- The fixed behavior score is zero because team.missingExpectedRoles contains image-generator, although ensure_gezel created Kseniya from illustrator, both delegation receipts identify her as image-generator, and her generation call succeeded. missingScenarioRoles in evals/src/bin/score-trial.ts compares display-role strings and does not recognize Illustrator as the capability. Preserve fixed 7.0; treat this as a scorer alias mismatch, not an absent image specialist.\n- At 320px, opening navigation expands scroll width to 397px. The cart is 360px wide and extends 40px off-screen; its overlay has no exposed close area, there is no close button, and Escape does not close it. These regressions are absent from the one-assertion fixed runtime gate (outcome.reason; artifacts.htmlFiles; advisory-runtime-check.json).\n- The logo contains generated nonsense lettering rather than the shop name. The contact form resets input and says a reply will arrive within 24 hours, but its source only simulates sending and makes no network request. The visible promise is misleading even for a static demo; checkout itself was not requested and is not penalized (artifacts.imageFiles; HTML source; advisory runtime checks).\n- Delivery required 42 tool calls and 14 write_file calls over 62m55s. Multiple HTML owners and repeated delegation contributed coordination overhead. The parked handoff explicitly instructed the delegator to end its turn, yet another delegation followed in that turn (toolUse; outcome.durationMs: 3774632; recorded handoff results).\n\n## Performance and GPU fit\n\nElapsed 62.91 minutes; 29522 generated tokens at 8.97 tok/s, prefill 382.97 tok/s. Peak VRAM 10654/12227 MiB; process-tree RSS 15467 MiB; whole-system RAM 31242/31865 MiB. Values come from perf.derived, perf.gpu, perf.process and perf.systemMemory; whole-system memory includes other applications.\n\nNative 0.1.57 CUDA, 65,536-token context, one slot, f16 KV, catalog reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. Shared GPU memory and paging were not measured. Two Saluki launches bracket a successful SDXL generation. Admission waited for capacity, intentionally terminated idle Saluki with expectedReason=memory-pressure, ran the image engine, then intentionally released SDXL and restored Saluki. This demonstrates sequential multimodal operation, not simultaneous model residency. Keep performance, fixed capability and advisory quality distinct.\n\n## Tactical experiment — not applied\n\nIn a proposed generic packages/core/src/model-profile/behaviors/prompt-verify-artifact-behavior.ts block, request one desktop/mobile interaction check and explicit demo labeling for local-only submission controls before declaring a web artifact ready. Gate the instruction on available browser tools; no task names, expected selectors or brand wording. A/B it on a storefront and an unrelated form/dashboard, measuring observed control failures and extra calls.\n\n## Strategic experiment — not applied\n\nIn evals/src/bin/score-trial.ts missingScenarioRoles, resolve canonical template/capability identity rather than matching user-facing role strings. In packages/service/src/chat/manager.ts, investigate returning the provider slot promptly after parked delegations and consolidating ownership of a shared output. In packages/mcp/src/validate.ts, add reusable advisory checks for mobile overflow and dismissible overlays. Measure these separately; do not change this frozen campaign.\n\n## Generalization and recommended next experiment\n\nRepetition 1 passed fixed 7.5 with advisory 2.25 and missing CSS. Repetition 2 has substantially better rendered output despite fixed 7.0, whose drop is driven by the role-name mismatch. SDXL succeeded through deliberate engine replacement, not simultaneous residency. Compare unrelated schema migration: both show that a fixed gate can disagree with broader observed quality, for different reasons.\n\nThe experiments above target reusable contract verification, state transitions, ownership or execution cost. Run each proposed change separately against the named task family and an unrelated task before recommending it; preserve baseline scores and avoid scenario-specific model hints. No prompt, tuning, source or gate change was applied.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/facts.json): pass, fixed 7.5, 81.05 min, 5.54 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/facts.json): pass, fixed 8.5, 2.46 min, 132.33 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-11-10-713Z-qnti/facts.json): pass, fixed 8.5, 1.88 min, 129.83 tok/s.\n\nSame-campaign repetitions share the frozen identity; older runs with other models/releases are contextual only. No causal native-version improvement is claimed.\n\n## Strategic note\n\nRole-label scoring and actual specialist execution disagree. The image is real, but broader UI behavior still needs review.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-engine-evidence.json, historical-comparison.json and the saved browser/contract checks where applicable.\n" }, { "cellId": "r2-saluki-cuda-core-tankcombat", "baselineCellId": "r2-saluki-cuda-core-tankcombat", "suite": "core", "scenario": "tankcombat", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T11:05:34.375Z", "finishedAt": "2026-11-05T11:13:06.751Z", "durationMs": 452376, "success": true, "reason": "tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 438327 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 8469, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": 3, "runtimeFailed": 0 } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 8469 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.028587649682359446 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 438327 }, { "source": "facts", "path": "outcome.durationMs", "value": 452376 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 74, "peakRssMb": 10674, "peakCpuPercent": 1239.8, "firstAtMs": 0, "lastAtMs": 432537 }, "gpu": { "available": true, "sampleCount": 74, "peakUtilPercent": 91, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 74, "peakUsedMb": 28072, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12210, "totalOutputTokens": 3484, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7474, "todayTokensCached": 7114, "todayTokensOut": 127, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7474, "totalTokensCached": 7114, "totalTokensOut": 127, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7474, "cachedInputTokens": 7114, "outputTokens": 127, "cost": 0, "durationMs": 91839, "outputTokensPerSec": 8.304880877359253, "at": "2026-11-05T11:07:16.233Z" }, "lastUpdated": "2026-11-05T11:07:16.233Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:12:56.889Z" } }, "source": "engine-log" }, "derived": { "durationMs": 436311, "meanTokensPerSec": 8.92, "genTokensPerSec": 8.92, "genTokens": 3484, "promptTokensPerSec": 589.69, "peakRssMb": 10674, "peakGpuUtilPercent": 91, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7474, "output": 127 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tank-combat-arcade/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-05T11:06:59.349Z", "completedAt": "2026-11-05T11:12:53.411Z", "ms": 354062, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "10b3de13-ec31-4d9a-9100-002cc027e013" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:05:40.125Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:05:40.016Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 354062 }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "tsumugi": 2, "zara": 1 }, "perTask": { "tank-combat-arcade/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.09539794921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.09539794921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "durationMs": 452376, "timeoutMs": 15824176, "budgetUsedFraction": 0.028587649682359446 }, "timing": { "startedAt": "2026-11-05T11:05:34.375Z", "finishedAt": "2026-11-05T11:13:06.751Z", "timeToFirstArtifactMs": 438327, "timeToLastArtifactWriteMs": 438371, "timeToFirstTokenMs": 28901, "firstTurnTtftMs": 18882, "timeToFirstToolCallMs": 85035 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "start_work": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/tank-combat-arcade/index.html", "finalBytes": 8469, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 439671, "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 8469, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null } ], "latest": { "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 8469, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": 3, "runtimeFailed": 0 } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 440106, "kind": "structured", "gezel": "zara", "question": "Your web page is finished. - **`index.html`**", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-05T11:05:34.379Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "[2026-11-05T11:05:34.379Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T11:05:34.379Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-tankcombat-xPD9Fa", "[2026-11-05T11:05:34.378Z] [trial] id=tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7 scenario=tankcombat model=underdog-saluki-27b-iq2-mix", "[2026-11-05T11:05:34.379Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T11:05:34.381Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T11:05:34.384Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T11:05:34.390Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-tankcombat-xPD9Fa", "[2026-11-05T11:05:34.390Z] [trial] throughput-scaled ceiling: 120m → 264m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T11:05:40.110Z] [trial] daemon spawned pid=50444 port=https://127.0.0.1:63479", "[2026-11-05T11:05:41.427Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T11:05:41.443Z] [trial] meester=zara", "[2026-11-05T11:05:41.588Z] [trial] sent prompt to meester (1688 chars)", "[2026-11-05T11:05:41.641Z] [poll] initial digests hard=d308582fa01fae3e soft=cd3e32596506df27", "[2026-11-05T11:07:04.114Z] [poll] hard-progress: d308582fa01fae3e → 8f236b5be9992cc1 after 82s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=157,stream=574) sniff=none)", "[2026-11-05T11:07:21.691Z] [poll] hard-progress: 8f236b5be9992cc1 → 4453d4073101129a after 18s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=1,slot=192,stream=673) sniff=none)", "[2026-11-05T11:11:24.499Z] [poll] hard-progress: 4453d4073101129a → a4a8761cbf63a32d after 243s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=1,slot=882,stream=4552) sniff=none)", "[2026-11-05T11:12:56.517Z] [poll] terminal: success=true reason=tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "[2026-11-05T11:12:56.941Z] [capture] dumped 3 session(s)", "[2026-11-05T11:12:56.944Z] [capture] snapshotted telemetry for 3 session(s)", "[2026-11-05T11:12:57.022Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T11:13:06.749Z] [trial] daemon shut down", "[2026-11-05T11:13:06.759Z] [trial] PASS duration=452376ms reason=tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))" ] }, "metrics": { "process": { "sampleCount": 74, "peakRssMb": 10674, "peakCpuPercent": 1239.8, "firstAtMs": 0, "lastAtMs": 432537 }, "gpu": { "available": true, "sampleCount": 74, "peakUtilPercent": 91, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 74, "peakUsedMb": 28072, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12210, "totalOutputTokens": 3484, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7474, "todayTokensCached": 7114, "todayTokensOut": 127, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7474, "totalTokensCached": 7114, "totalTokensOut": 127, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7474, "cachedInputTokens": 7114, "outputTokens": 127, "cost": 0, "durationMs": 91839, "outputTokensPerSec": 8.304880877359253, "at": "2026-11-05T11:07:16.233Z" }, "lastUpdated": "2026-11-05T11:07:16.233Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:12:56.889Z" } }, "source": "engine-log" }, "derived": { "durationMs": 436311, "meanTokensPerSec": 8.92, "genTokensPerSec": 8.92, "genTokens": 3484, "promptTokensPerSec": 589.69, "peakRssMb": 10674, "peakGpuUtilPercent": 91, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7474, "output": 127 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T11:05:34.375Z", "finishedAt": "2026-11-05T11:13:06.751Z", "timeToFirstArtifactMs": 438327, "timeToLastArtifactWriteMs": 438371, "timeToFirstTokenMs": 28901, "firstTurnTtftMs": 18882, "timeToFirstToolCallMs": 85035 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tank-combat-arcade/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-05T11:06:59.349Z", "completedAt": "2026-11-05T11:12:53.411Z", "ms": 354062, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "10b3de13-ec31-4d9a-9100-002cc027e013" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:05:40.125Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:05:40.016Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 354062 }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "tsumugi": 2, "zara": 1 }, "perTask": { "tank-combat-arcade/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.09539794921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.09539794921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-tankcombat", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "scenarioId": "tankcombat", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/workspace/tank-combat-arcade/index.html", "artifactBytes": 8469, "artifactSha256": "5b9b9f429bf032141c4fc6e7d918b61a2726528a5dbdfd7c264a73bf7f6b23f6", "axisSource": "evals/src/llm-judge.ts DEFAULT_HTML_AXES", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 4, "functionalCompleteness": 1, "codeQuality": 2, "polish": 2 }, "meanScore": 2.25, "justification": "The grid and tanks render, movement and restart respond, but combat is broken. Space produces no shots because cooldown is undefined. Fire arguments are shifted, timers mix seconds and milliseconds, and destroying an enemy converts its array to NaN. The enemy counter displays [object Object].", "reviewBasis": "Read all 8,469 HTML bytes and inspected the rendered game. Used real keyboard input for Space, D and R; separately forced enemy-fire/collision state in browser memory to inspect latent branches. These branch probes do not alter the source or fixed result.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/workspace/tank-combat-arcade/index.html", "bytes": 8469, "sha256": "5b9b9f429bf032141c4fc6e7d918b61a2726528a5dbdfd7c264a73bf7f6b23f6" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:24.378Z", "reviewId": "046", "cellId": "r2-saluki-cuda-core-tankcombat", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "scenarioId": "tankcombat", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "baselineCellId": "r2-saluki-cuda-core-tankcombat", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "829949d8efe47612cbda1d2366410685d0d0ecf9fdb2068acc940bf92406356c", "evidenceSha256": "b3b0eaed975396e8c6af0db5902eda8bbf002f82e1f1350a62a7195bbc44fc68", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/workspace/tank-combat-arcade/index.html", "bytes": 8469, "sha256": "5b9b9f429bf032141c4fc6e7d918b61a2726528a5dbdfd7c264a73bf7f6b23f6", "shownBytes": 8469, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 4, "functionalCompleteness": 2, "codeQuality": 2, "polish": 2 }, "justification": "Movement, HUD and inputTick work, but shooting is broken. player.fireCooldown starts undefined, so `undefined <= 0` is false and the player never fires. fire() is called with a mismatched signature, so bullets get wrong values and enemy bullets don't move. The Enemies HUD shows an array, and `enemies++` corrupts state.", "limitations": "Source-only review from reading the code; no rendering or runtime execution. Conclusions about the bugs come from static analysis and were not run in a browser.", "meanScore": 2.5, "durationMs": 19740, "costUsd": 0.0368736, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6516, "cache_read_input_tokens": 856, "output_tokens": 1072, "output_tokens_details": { "thinking_tokens": 752 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6516, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 1072, "cache_read_input_tokens": 856, "cache_creation_input_tokens": 6516, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6516 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 1072, "cacheReadInputTokens": 856, "cacheCreationInputTokens": 6516, "webSearchRequests": 0, "costUSD": 0.0368736, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 752, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6516, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - tankcombat / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 7m 32s; budget used: 2.9%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 438327`; `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":8469,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"game...`; `facts.sniff.latest.bytes: 8469` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.028587649682359446`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 438327`; `facts.outcome.durationMs: 452376` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10674 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 91% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28072 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.92 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 12210 + 3484 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 354s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 10% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 438327`\n- `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":8469,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"game...`\n- `facts.sniff.latest.bytes: 8469`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.028587649682359446`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 438327`\n- `facts.outcome.durationMs: 452376`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 438327`; `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":8469,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"game...`; `facts.sniff.latest.bytes: 8469`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.028587649682359446`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 438327`; `facts.outcome.durationMs: 452376`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is preserved.\n\n| Axis | Advisory score |\n|---|---:|\n| visualQuality | 4/10 |\n| functionalCompleteness | 1/10 |\n| codeQuality | 2/10 |\n| polish | 2/10 |\n| Mean | 2.25/10 |\n\nThe grid and tanks render, movement and restart respond, but combat is broken. Space produces no shots because cooldown is undefined. Fire arguments are shifted, timers mix seconds and milliseconds, and destroying an enemy converts its array to NaN. The enemy counter displays [object Object].\n\nRead all 8,469 HTML bytes and inspected the rendered game. Used real keyboard input for Space, D and R; separately forced enemy-fire/collision state in browser memory to inspect latent branches. These branch probes do not alter the source or fixed result.\n\n## Positives\n\n- A self-contained game was delivered in one write and passed the three fixed browser assertions in 7m32s (toolUse.byTool.write_file: 1; outcome.durationMs: 452376; outcome.reason).\n- Real D input moves the tank, and R resets position, score and lives; the canvas and colors are readable (artifacts.htmlFiles; advisory-runtime-check.json).\n\n## Negatives\n\n- Space increments inputTick but produces zero bullets: player.fireCooldown is never initialized, so undefined <= 0 is false. Keyboard-event activity is not proof that the advertised action works (sniff.latest.signals: keyboard-input; outcome.success; advisory-runtime-check.afterFire).\n- fire declares six parameters but both callers supply five, shifting position/angle/speed and leaving fromPlayer undefined. A forced enemy-fire probe produced a stationary malformed projectile. Millisecond-named cooldown values are decremented by seconds, and movement is only about 3.2 pixels per second (artifacts.htmlFiles source; advisory-runtime-check.forcedEnemyFire and movement).\n- The enemies field is both an array and a counter. The HUD displays [object Object]; after a deliberately injected valid hit, enemies++ converts the array to NaN and draw throws TypeError. The forced branch illustrates a latent defect, not a naturally reached kill during the short smoke test (artifacts.htmlFiles; advisory-runtime-check.forcedCollision).\n\n## Performance and GPU fit\n\nElapsed 7.54 minutes; 3484 generated tokens at 8.92 tok/s, prefill 589.69 tok/s. Peak VRAM 10626/12227 MiB; process-tree RSS 10674 MiB; whole-system RAM 28072/31865 MiB. Values come from perf.derived, perf.gpu, perf.process and perf.systemMemory; whole-system memory includes other applications.\n\nNative 0.1.57 CUDA, 65,536-token context, one slot, f16 KV, catalog reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. Shared GPU memory and paging were not measured. One Saluki launch; no explicit CUDA OOM, context overflow or unexpected native exit found. Keep performance, fixed capability and advisory quality distinct.\n\n## Tactical experiment — not applied\n\nIn a proposed generic packages/core/src/model-profile/behaviors/prompt-verify-artifact-behavior.ts, request action-to-state checks, initialized state, consistent time units and one end/reset cycle for interactive outputs. Keep examples and wording independent of specific games. Compare this game with an unrelated timer/form; count behavioral defects and extra tool cost. No sampling change is justified by the clean single-write trace.\n\n## Strategic experiment — not applied\n\nExtend reusable artifact diagnostics in packages/mcp/src/validate.ts to check callback argument arity, uninitialized state and mixed collection/counter usage, while using browser evidence for behavior. Keep anchored gates unchanged during this campaign. For later evaluator work, report control activation and resulting state separately rather than interpreting inputTick as gameplay.\n\n## Generalization and recommended next experiment\n\nBoth repetitions scored fixed 10.0 while combat was unusable. Repetition 1 had key/code mismatches and unit errors; repetition 2 fixes D/R input but introduces undefined cooldown, argument and collection defects. Its advisory mean is 2.25 versus 3.25. The unrelated second tic-tac-toe passed independent win/draw/reset checks at advisory 6.0, so complete HTML generation is reliable on some tasks while behavioral verification remains uneven.\n\nThe experiments above target reusable contract verification, state transitions, ownership or execution cost. Run each proposed change separately against the named task family and an unrelated task before recommending it; preserve baseline scores and avoid scenario-specific model hints. No prompt, tuning, source or gate change was applied.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/facts.json): pass, fixed 10, 30.01 min, 2.89 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/facts.json): pass, fixed 10, 0.86 min, 133.35 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-16-33-999Z-43cw/facts.json): pass, fixed 10, 0.95 min, 129.87 tok/s.\n\nSame-campaign repetitions share the frozen identity; older runs with other models/releases are contextual only. No causal native-version improvement is claimed.\n\n## Strategic note\n\nThe core firing failure recurs in both repetitions with different immediate causes. Fixed 10.0 does not establish playable combat.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-engine-evidence.json, historical-comparison.json and the saved browser/contract checks where applicable.\n" }, { "cellId": "r2-saluki-cuda-core-schema-migration", "baselineCellId": "r2-saluki-cuda-core-schema-migration", "suite": "core", "scenario": "schema-migration", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T11:15:55.019Z", "finishedAt": "2026-11-05T11:24:43.445Z", "durationMs": 528426, "success": true, "reason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 72084 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "schema-migration", "bytes": 7927, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.1144922904589758 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 17 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 72084 }, { "source": "facts", "path": "outcome.durationMs", "value": 528426 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 88, "peakRssMb": 15389, "peakCpuPercent": 1198.6, "firstAtMs": 0, "lastAtMs": 514379 }, "gpu": { "available": true, "sampleCount": 88, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 88, "peakUsedMb": 31086, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 57102, "totalOutputTokens": 3111, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 44181, "todayTokensCached": 4924, "todayTokensOut": 2924, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 44181, "totalTokensCached": 4924, "totalTokensOut": 2924, "totalCost": 0, "medianOutputTokensPerSec": 9.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.2, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10144, "outputTokens": 1089, "cost": 0, "durationMs": 179868, "outputTokensPerSec": 9.167179441723828, "at": "2026-11-05T11:24:34.009Z" }, "lastUpdated": "2026-11-05T11:24:34.009Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:24:37.151Z" } }, "source": "engine-log" }, "derived": { "durationMs": 516334, "meanTokensPerSec": 9.24, "genTokensPerSec": 9.24, "genTokens": 3111, "promptTokensPerSec": 415.23, "peakRssMb": 15389, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44181, "output": 2924 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:16:00.822Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:16:00.734Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "riley": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 14, "maxContextFill": 0.1608734130859375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1608734130859375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "durationMs": 528426, "timeoutMs": 4615385, "budgetUsedFraction": 0.1144922904589758 }, "timing": { "startedAt": "2026-11-05T11:15:55.019Z", "finishedAt": "2026-11-05T11:24:43.445Z", "timeToFirstArtifactMs": 72084, "timeToLastArtifactWriteMs": 518990, "timeToFirstTokenMs": 23581, "firstTurnTtftMs": 14118, "timeToFirstToolCallMs": 31335 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 17, "byTool": { "read_files": 1, "write_file": 4, "replace_in_file": 8, "read_file": 3, "stat": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 11 }, "sniff": { "progression": [ { "atMs": 7209, "filePath": "schema-migration", "bytes": 1659, "score": 0, "scoreMax": 7, "signals": [], "failReason": "types.ts still declares name or is missing required string firstName/lastName fields on User and CreateUserInput" }, { "atMs": 77842, "filePath": "schema-migration", "bytes": 2686, "score": 0, "scoreMax": 7, "signals": [], "failReason": "types.ts still declares name or is missing required string firstName/lastName fields on User and CreateUserInput" }, { "atMs": 101955, "filePath": "schema-migration", "bytes": 2733, "score": 2, "scoreMax": 7, "signals": [ "types-updated", "migrate-function" ], "failReason": "store.ts still uses a live name property or is missing firstName/lastName" }, { "atMs": 142525, "filePath": "schema-migration", "bytes": 2819, "score": 3, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "migrate-function" ], "failReason": "handlers.ts unmet subcontracts: format-display-full-name (return firstName + space + lastName from formatDisplayName); render-card-full-name (include firstName + space + lastName in the rendered card); summary-full-name (include each mapped user firstName + space + lastName); no-legacy-name-access (remove every live .name property access)" }, { "atMs": 183729, "filePath": "schema-migration", "bytes": 2887, "score": 4, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function" ], "failReason": "tests/migrate.test.ts not present yet" }, { "atMs": 261425, "filePath": "schema-migration", "bytes": 2839, "score": 4, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function" ], "failReason": "tests/migrate.test.ts not present yet" }, { "atMs": 344730, "filePath": "schema-migration", "bytes": 4119, "score": 5, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present" ], "failReason": "MIGRATION.md not present yet" }, { "atMs": 521621, "filePath": "schema-migration", "bytes": 7927, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } ], "latest": { "filePath": "schema-migration", "bytes": 7927, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T11:15:55.022Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "[2026-11-05T11:15:55.022Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-schema-migration-tME95K", "[2026-11-05T11:15:55.022Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T11:15:55.022Z] [trial] id=schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou scenario=schema-migration model=underdog-saluki-27b-iq2-mix", "[2026-11-05T11:15:55.022Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T11:15:55.024Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T11:15:55.029Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T11:15:55.034Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-schema-migration-tME95K", "[2026-11-05T11:15:55.034Z] [trial] throughput-scaled ceiling: 35m → 77m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T11:16:00.787Z] [trial] daemon spawned pid=40092 port=https://127.0.0.1:52951", "[2026-11-05T11:16:01.673Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T11:16:01.689Z] [trial] running scenario setup hook", "[2026-11-05T11:16:01.689Z] [trial] meester=rafael", "[2026-11-05T11:16:02.129Z] [trial] scenario setup complete", "[2026-11-05T11:16:02.131Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T11:16:02.197Z] [poll] initial digests hard=76f0da89e72710b4 soft=e2e7cadb8ce2ff5f", "[2026-11-05T11:16:02.373Z] [poll] hard-progress: 76f0da89e72710b4 → 51248212490e818f after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/1659B)", "[2026-11-05T11:16:31.386Z] [poll] hard-progress: 51248212490e818f → 53c4128aa2d6c333 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=27,stream=88) sniff=0/1659B)", "[2026-11-05T11:17:12.940Z] [poll] hard-progress: 53c4128aa2d6c333 → a7a484b091b5d769 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=2,slot=132,stream=634) sniff=0/2686B)", "[2026-11-05T11:17:37.093Z] [poll] hard-progress: a7a484b091b5d769 → 8558e99e808bef88 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=3,slot=182,stream=862) sniff=2/2733B)", "[2026-11-05T11:18:17.665Z] [poll] hard-progress: 8558e99e808bef88 → 8f01282292cc6922 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=5,slot=240,stream=1098) sniff=3/2819B)", "[2026-11-05T11:18:58.855Z] [poll] hard-progress: 8f01282292cc6922 → 215b49898adac0e2 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=5,tools=8,slot=316,stream=1450) sniff=4/2887B)", "[2026-11-05T11:19:10.808Z] [poll] hard-progress: 215b49898adac0e2 → 78721d1ba6329724 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=5,tools=9,slot=324,stream=1468) sniff=4/2887B)", "[2026-11-05T11:20:16.541Z] [poll] hard-progress: 78721d1ba6329724 → 1dec40b5e664f779 after 66s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=6,tools=12,slot=460,stream=2102) sniff=4/2839B)", "[2026-11-05T11:20:27.558Z] [poll] hard-progress: 1dec40b5e664f779 → f24dfefc77fea4a6 after 11s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=6,tools=13,slot=471,stream=2122) sniff=4/2839B)" ] }, "metrics": { "process": { "sampleCount": 88, "peakRssMb": 15389, "peakCpuPercent": 1198.6, "firstAtMs": 0, "lastAtMs": 514379 }, "gpu": { "available": true, "sampleCount": 88, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 88, "peakUsedMb": 31086, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 57102, "totalOutputTokens": 3111, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 44181, "todayTokensCached": 4924, "todayTokensOut": 2924, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 44181, "totalTokensCached": 4924, "totalTokensOut": 2924, "totalCost": 0, "medianOutputTokensPerSec": 9.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.2, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10144, "outputTokens": 1089, "cost": 0, "durationMs": 179868, "outputTokensPerSec": 9.167179441723828, "at": "2026-11-05T11:24:34.009Z" }, "lastUpdated": "2026-11-05T11:24:34.009Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:24:37.151Z" } }, "source": "engine-log" }, "derived": { "durationMs": 516334, "meanTokensPerSec": 9.24, "genTokensPerSec": 9.24, "genTokens": 3111, "promptTokensPerSec": 415.23, "peakRssMb": 15389, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 44181, "output": 2924 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T11:15:55.019Z", "finishedAt": "2026-11-05T11:24:43.445Z", "timeToFirstArtifactMs": 72084, "timeToLastArtifactWriteMs": 518990, "timeToFirstTokenMs": 23581, "firstTurnTtftMs": 14118, "timeToFirstToolCallMs": 31335 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:16:00.822Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:16:00.734Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "riley": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 14, "maxContextFill": 0.1608734130859375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1608734130859375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-schema-migration", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "scenarioId": "schema-migration", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/src/types.ts", "artifactBytes": 422, "artifactSha256": "00844023ad6b9c14cc8d3680b9e289d9eff42196060a2b0f54b004ec01fbddea", "axisSource": "evals/src/scenarios/schema-migration.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. Types, store, three handlers, three migration cases and documentation were delivered; seven fixed checks pass. However, migrateUser returns only id/email/name parts and drops other input fields despite the preservation promise in the brief and documentation. An independent extra-field probe confirms the loss.", "reviewBasis": "Read the six authored/modified deliverables in full, the mission objectives and fixed gate evidence. Ran a separate pure-function probe with extra non-name fields. Each textual artifact is below 20KiB; no numeric axes were invented.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/src/types.ts", "bytes": 422, "sha256": "00844023ad6b9c14cc8d3680b9e289d9eff42196060a2b0f54b004ec01fbddea" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/src/store.ts", "bytes": 824, "sha256": "abbcac468914e729812abdb5639e3eccee02ee9e9b949057c92c6f1125c23356" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/src/handlers.ts", "bytes": 614, "sha256": "f151082616220d177a6e1bc1d525727b820376df24996db8e901924a3d2b1b85" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/src/migrate.ts", "bytes": 979, "sha256": "0aa910851dea95a48b8d8f1829a82726ae7b9bb5fcac35c3dd489dbd1069f8e3" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/tests/migrate.test.ts", "bytes": 1280, "sha256": "ea7fae814e812ac87130d2230c2641df9819f6b3ab3e858e0a6a996cf4a579dc" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/MIGRATION.md", "bytes": 3812, "sha256": "e9e59bee334f7cf44f5d05129b498f21e2714e832d5e9c500b877968d733fdbb" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:27.229Z", "reviewId": "047", "cellId": "r2-saluki-cuda-core-schema-migration", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "scenarioId": "schema-migration", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "baselineCellId": "r2-saluki-cuda-core-schema-migration", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "c368579cccbe40318f66379f4603e352b07d2d08b98432cc082bafc01cba7b81", "evidenceSha256": "c20fe6ed03b509e51eb70f8e6a803cc90a707fb9bd10c3af96bcba7bcaffc493", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/src/types.ts", "bytes": 422, "sha256": "00844023ad6b9c14cc8d3680b9e289d9eff42196060a2b0f54b004ec01fbddea", "shownBytes": 422, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "Only src/types.ts is shown: User and CreateUserInput correctly split name into firstName/lastName, with a migration comment. The other five required files (store.ts, handlers.ts, migrate.ts, migrate.test.ts, MIGRATION.md) aren't in the evidence, so completeness can't be judged.", "limitations": "Evidence is a single artifact; no other files, transcript or typecheck results were available. Whether migrateUser, the tests, MIGRATION.md or the store/handler updates exist or are correct is unknown. No runtime success is inferred.", "meanScore": null, "durationMs": 14124, "costUsd": 0.010398000000000001, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1816, "cache_read_input_tokens": 700, "output_tokens": 306, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1816, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 306, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1816, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1816 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 306, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1816, "webSearchRequests": 0, "costUSD": 0.010398000000000001, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1816, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - schema-migration / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 8m 48s; budget used: 11.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 72084`; `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":7927,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.1144922904589758`; `facts.toolUse.totalToolCalls: 17`; `facts.timing.timeToFirstArtifactMs: 72084`; `facts.outcome.durationMs: 528426` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15389 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31086 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.24 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 57102 + 3111 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 16% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 72084`\n- `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":7927,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.1144922904589758`\n- `facts.toolUse.totalToolCalls: 17`\n- `facts.timing.timeToFirstArtifactMs: 72084`\n- `facts.outcome.durationMs: 528426`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 72084`; `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":7927,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.1144922904589758`; `facts.toolUse.totalToolCalls: 17`; `facts.timing.timeToFirstArtifactMs: 72084`; `facts.outcome.durationMs: 528426`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 9.5/10** is preserved.\n\nNo declared axes; prose review only. No numeric quality score was invented.\n\nNo declared axes. Types, store, three handlers, three migration cases and documentation were delivered; seven fixed checks pass. However, migrateUser returns only id/email/name parts and drops other input fields despite the preservation promise in the brief and documentation. An independent extra-field probe confirms the loss.\n\nRead the six authored/modified deliverables in full, the mission objectives and fixed gate evidence. Ran a separate pure-function probe with extra non-name fields. Each textual artifact is below 20KiB; no numeric axes were invented.\n\n## Positives\n\n- All seven fixed acceptance signals passed, including strict compilation, migration helper, meaningful test cases and documentation. The code consistently updates both required name fields across the store and three handlers (outcome.success; sniff.latest.score: 7/7; artifacts.otherFileCount).\n- The test cases call the real helper with full legacy records and verify id/email plus normal, one-word and multi-word splits. The code preserves the caller input object (artifacts.otherFileCount; authored tests; advisory-contract-check.inputUnchanged).\n\n## Negatives\n\n- The helper discards arbitrary non-name properties. With active:true and tags:[customer], the returned object loses both, contradicting mission objective 4 and MIGRATION.md. The hidden gate covers seeded id/email fields but does not establish the broader promise (outcome.success; sniff.latest; artifact source; advisory-contract-check.nonNameFieldsPreserved:false).\n- The same fixed score 9.5 hides a substantive difference: repetition 1 used {name,...rest} and preserved additional properties, while repetition 2 constructs only four output fields. Stale comments also still describe the old single-name schema (artifacts.otherFileCount; both preserved migrate.ts files).\n\n## Performance and GPU fit\n\nElapsed 8.81 minutes; 3111 generated tokens at 9.24 tok/s, prefill 415.23 tok/s. Peak VRAM 10640/12227 MiB; process-tree RSS 15389 MiB; whole-system RAM 31086/31865 MiB. Values come from perf.derived, perf.gpu, perf.process and perf.systemMemory; whole-system memory includes other applications.\n\nNative 0.1.57 CUDA, 65,536-token context, one slot, f16 KV, catalog reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. Shared GPU memory and paging were not measured. One Saluki launch; no explicit CUDA OOM, context overflow or unexpected native exit found. Keep performance, fixed capability and advisory quality distinct.\n\n## Tactical experiment — not applied\n\nIn a proposed generic packages/core/src/model-profile/behaviors/prompt-verify-artifact-behavior.ts, instruct code authors to turn each explicit preservation promise into one adversarial example containing an additional input field. Use a generic contract check, not seeded fixture names. A/B on this migration and an unrelated data transformation; measure property retention and token/call cost.\n\n## Strategic experiment — not applied\n\nIn evals/src/scenarios/schema-migration.ts, a future separately versioned gate should match the stated preservation contract with an extra-property case. Generalize this in contract-based artifact validation rather than inserting the expected record into model prompts. In generated documentation checks, compare strong preservation claims to executable evidence. Leave this run and its fixed 9.5 untouched.\n\n## Generalization and recommended next experiment\n\nRepetition 1 also scored 9.5 and took 27m28s at 2.79 tok/s; repetition 2 took 8m48s at 9.24 tok/s with the same 17 calls. Its extra-field preservation regressed even though the fixed score stayed level. Tankcombat independently demonstrates a semantic miss hidden by a passing gate. Higher current throughput is observed under the same placement, not proof of a tuning or native-version improvement.\n\nThe experiments above target reusable contract verification, state transitions, ownership or execution cost. Run each proposed change separately against the named task family and an unrelated task before recommending it; preserve baseline scores and avoid scenario-specific model hints. No prompt, tuning, source or gate change was applied.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/facts.json): pass, fixed 9.5, 27.47 min, 2.79 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/facts.json): pass, fixed 9, 3.49 min, 118.49 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-21-10-691Z-kbc7/facts.json): pass, fixed 9.5, 1.90 min, 123.62 tok/s.\n\nSame-campaign repetitions share the frozen identity; older runs with other models/releases are contextual only. No causal native-version improvement is claimed.\n\n## Strategic note\n\nThe extra-field preservation behavior regressed relative to repetition 1 while both fixed scores remained 9.5. Preserve this as an advisory semantic regression.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-engine-evidence.json, historical-comparison.json and the saved browser/contract checks where applicable.\n" }, { "cellId": "r2-saluki-cuda-core-failing-tests-spec", "baselineCellId": "r2-saluki-cuda-core-failing-tests-spec", "suite": "core", "scenario": "failing-tests-spec", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T11:27:25.397Z", "finishedAt": "2026-11-05T11:29:11.608Z", "durationMs": 106211, "success": true, "reason": "vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 98968 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "failing-tests-spec", "bytes": 1558, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.03221733956622723 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 98968 }, { "source": "facts", "path": "outcome.durationMs", "value": 106211 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 17, "peakRssMb": 9882, "peakCpuPercent": 1197.8, "firstAtMs": 1, "lastAtMs": 94191 }, "gpu": { "available": true, "sampleCount": 17, "peakUtilPercent": 99, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 26397, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 8534, "totalOutputTokens": 539, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4907, "todayTokensCached": 0, "todayTokensOut": 482, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4907, "totalTokensCached": 0, "totalTokensOut": 482, "totalCost": 0, "medianOutputTokensPerSec": 8.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.2, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 4907, "outputTokens": 482, "cost": 0, "durationMs": 88375, "outputTokensPerSec": 8.219513997738352, "at": "2026-11-05T11:29:04.374Z" }, "lastUpdated": "2026-11-05T11:29:04.374Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:29:08.392Z" } }, "source": "engine-log" }, "derived": { "durationMs": 96727, "meanTokensPerSec": 8.27, "genTokensPerSec": 8.27, "genTokens": 539, "promptTokensPerSec": 570.91, "peakRssMb": 9882, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 4907, "output": 482 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:27:31.247Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:27:31.152Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "priya": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.0513458251953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0513458251953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "durationMs": 106211, "timeoutMs": 3296703, "budgetUsedFraction": 0.03221733956622723 }, "timing": { "startedAt": "2026-11-05T11:27:25.397Z", "finishedAt": "2026-11-05T11:29:11.608Z", "timeToFirstArtifactMs": 98968, "timeToLastArtifactWriteMs": 98978, "timeToFirstTokenMs": 25165, "firstTurnTtftMs": 14563, "timeToFirstToolCallMs": 31178 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 3, "byTool": { "read_file": 2, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 10681, "filePath": "failing-tests-spec", "bytes": 402, "score": 2, "scoreMax": 4, "signals": [ "tests-untouched", "tsc-clean" ], "failReason": "vitest run failed (exit 1): × starts in \"draft\" 3ms | × starts with an empty history log 0ms | × submit moves draft to pending and returns the new state 0ms | × pay moves pending to paid 0ms | FAIL tests/machine.test.ts > order machine: initial state > starts in \"draft\" | Error: createOrderMachine is not implemented yet | FAIL tests/machine.test.ts > order machine: initial state > starts with an empty history log | FAIL tests/machine.test.ts > order machine: transitions > submit moves draft to pending and returns the new state" }, { "atMs": 102585, "filePath": "failing-tests-spec", "bytes": 1558, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } ], "latest": { "filePath": "failing-tests-spec", "bytes": 1558, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T11:27:25.400Z] [trial] id=failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk scenario=failing-tests-spec model=underdog-saluki-27b-iq2-mix", "[2026-11-05T11:27:25.400Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T11:27:25.400Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "[2026-11-05T11:27:25.400Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-failing-tests-spec-SNpdg2", "[2026-11-05T11:27:25.400Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T11:27:25.403Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T11:27:25.406Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T11:27:25.412Z] [trial] throughput-scaled ceiling: 25m → 55m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T11:27:25.412Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-failing-tests-spec-SNpdg2", "[2026-11-05T11:27:31.239Z] [trial] daemon spawned pid=46844 port=https://127.0.0.1:58271", "[2026-11-05T11:27:32.482Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T11:27:32.504Z] [trial] running scenario setup hook", "[2026-11-05T11:27:32.504Z] [trial] meester=senna", "[2026-11-05T11:27:32.913Z] [trial] scenario setup complete", "[2026-11-05T11:27:32.915Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T11:27:32.983Z] [poll] initial digests hard=43c8666eab7b061a soft=17aa919cb0cba15f", "[2026-11-05T11:27:36.157Z] [poll] hard-progress: 43c8666eab7b061a → 22314eb2bcac1f56 after 3s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=0,slot=0,stream=0) sniff=2/402B)", "[2026-11-05T11:27:56.752Z] [poll] hard-progress: 22314eb2bcac1f56 → 13446eaa4938eb0e after 21s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=18,stream=36) sniff=2/402B)", "[2026-11-05T11:29:07.982Z] [poll] terminal: success=true reason=vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "[2026-11-05T11:29:08.429Z] [capture] dumped 1 session(s)", "[2026-11-05T11:29:08.433Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-05T11:29:08.492Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T11:29:11.606Z] [trial] daemon shut down", "[2026-11-05T11:29:11.614Z] [trial] PASS duration=106211ms reason=vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)" ] }, "metrics": { "process": { "sampleCount": 17, "peakRssMb": 9882, "peakCpuPercent": 1197.8, "firstAtMs": 1, "lastAtMs": 94191 }, "gpu": { "available": true, "sampleCount": 17, "peakUtilPercent": 99, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 26397, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 8534, "totalOutputTokens": 539, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4907, "todayTokensCached": 0, "todayTokensOut": 482, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4907, "totalTokensCached": 0, "totalTokensOut": 482, "totalCost": 0, "medianOutputTokensPerSec": 8.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.2, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 4907, "outputTokens": 482, "cost": 0, "durationMs": 88375, "outputTokensPerSec": 8.219513997738352, "at": "2026-11-05T11:29:04.374Z" }, "lastUpdated": "2026-11-05T11:29:04.374Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:29:08.392Z" } }, "source": "engine-log" }, "derived": { "durationMs": 96727, "meanTokensPerSec": 8.27, "genTokensPerSec": 8.27, "genTokens": 539, "promptTokensPerSec": 570.91, "peakRssMb": 9882, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 4907, "output": 482 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T11:27:25.397Z", "finishedAt": "2026-11-05T11:29:11.608Z", "timeToFirstArtifactMs": 98968, "timeToLastArtifactWriteMs": 98978, "timeToFirstTokenMs": 25165, "firstTurnTtftMs": 14563, "timeToFirstToolCallMs": 31178 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:27:31.247Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:27:31.152Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "priya": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.0513458251953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0513458251953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-failing-tests-spec", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "scenarioId": "failing-tests-spec", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/workspace/machine-kata/src/machine.ts", "artifactBytes": 1558, "artifactSha256": "7d1a41e50124eab8690e29235d7f3f5a39fa830ee1c1c9281a3547f33150953e", "axisSource": "evals/src/scenarios/failing-tests-spec.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. All 17 untouched tests and strict compilation pass; normal transitions, history and the expedite guard work. However, inherited keys such as toString count as events: can returns true and send changes state to a function, after which later calls throw. The unknown-event contract is incomplete.", "reviewBasis": "Read the implementation and seeded test suite. Relied on the recorded standard Vitest/compiler gates, then ran one independent inherited-key probe against an in-memory transpilation of the unchanged artifact.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/workspace/machine-kata/src/machine.ts", "bytes": 1558, "sha256": "7d1a41e50124eab8690e29235d7f3f5a39fa830ee1c1c9281a3547f33150953e" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/workspace/machine-kata/tests/machine.test.ts", "bytes": 4926, "sha256": "12e2d36908178da95c12c9d5bf5d3bb7d8c67606f2ec4faa3ffd5fff62517997" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:30.369Z", "reviewId": "048", "cellId": "r2-saluki-cuda-core-failing-tests-spec", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "scenarioId": "failing-tests-spec", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "baselineCellId": "r2-saluki-cuda-core-failing-tests-spec", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "c007047017924e152c5330c2ee10b78d68eaa17cee66d3a4dda5175e57a607a3", "evidenceSha256": "63fd8cda405edc919e64e5c5305e9b1ad8c4bacfa6256d23707b6bebfd75f1c5", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/workspace/machine-kata/src/machine.ts", "bytes": 1558, "sha256": "7d1a41e50124eab8690e29235d7f3f5a39fa830ee1c1c9281a3547f33150953e", "shownBytes": 1558, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The machine.ts is coherent and typed: six states, a guarded expedite event, a history log, and a formatted invalid-transition error. It has a shallow-copy quirk, but no mutation occurs. I can't confirm it matches the tests' exact messages or semantics, since the test file isn't shown.", "limitations": "Tests/machine.test.ts and harness results weren't provided, so I can't verify exact error messages, guard semantics, or whether history should be returned as a copy. No runtime or tsc output was observed. Source-only review.", "meanScore": null, "durationMs": 14159, "costUsd": 0.012206000000000002, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2278, "cache_read_input_tokens": 700, "output_tokens": 302, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2278, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 302, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2278, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2278 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 302, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2278, "webSearchRequests": 0, "costUSD": 0.012206000000000002, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2278, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - failing-tests-spec / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 1m 46s; budget used: 3.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 98968`; `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1558,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.03221733956622723`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 98968`; `facts.outcome.durationMs: 106211` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9882 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 99% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26397 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.27 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 8534 + 539 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 5% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 98968`\n- `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1558,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.03221733956622723`\n- `facts.toolUse.totalToolCalls: 3`\n- `facts.timing.timeToFirstArtifactMs: 98968`\n- `facts.outcome.durationMs: 106211`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 98968`; `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1558,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.03221733956622723`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 98968`; `facts.outcome.durationMs: 106211`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is preserved.\n\nNo declared axes; prose review only. No numeric quality score was invented.\n\nNo declared axes. All 17 untouched tests and strict compilation pass; normal transitions, history and the expedite guard work. However, inherited keys such as toString count as events: can returns true and send changes state to a function, after which later calls throw. The unknown-event contract is incomplete.\n\nRead the implementation and seeded test suite. Relied on the recorded standard Vitest/compiler gates, then ran one independent inherited-key probe against an in-memory transpilation of the unchanged artifact.\n\n## Positives\n\n- The model read two files and wrote one implementation, passing the seeded 17 tests without modifying them and passing strict TypeScript (toolUse.totalToolCalls: 3; sniff.latest signals tests-untouched, machine-implemented, vitest-green, tsc-clean).\n- Delivery took 1m46s with 539 recorded generated tokens and no red flag or auto-answer. The ordinary transition paths, history behavior and expedite option match the explicit examples (outcome.durationMs: 106211; perf.derived.genTokens; toolUse.redFlags; autoAnswer.total; authored implementation versus tests).\n\n## Negatives\n\n- can uses event in transitions[state] and send indexes an ordinary object without an own-property check. toString is accepted, a function is returned and stored as state, and a later can call throws. This violates the arbitrary-string unknown-event contract despite all seeded examples passing (artifacts.otherFileCount; advisory-contract-check).\n- The public string event API and generic Record annotation conceal inherited properties at compile time; the seeded unknown case only tests refund. Compilation and example tests do not establish rejection of every unknown event (sniff.latest tsc-clean/vitest-green; authored src/machine.ts and unchanged test suite).\n\n## Performance and GPU fit\n\nElapsed 1.77 minutes; 539 generated tokens at 8.27 tok/s, prefill 570.91 tok/s. Peak VRAM 10626/12227 MiB; process-tree RSS 9882 MiB; whole-system RAM 26397/31865 MiB. Values come from perf.derived, perf.gpu, perf.process and perf.systemMemory; whole-system memory includes other applications.\n\nNative 0.1.57 CUDA, 65,536-token context, one slot, f16 KV, catalog reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. Shared GPU memory and paging were not measured. One Saluki launch; no explicit CUDA OOM, context overflow or unexpected native exit found. Keep performance, fixed capability and advisory quality distinct.\n\n## Tactical experiment — not applied\n\nFor generic table-driven code, add a contract-review prompt in proposed packages/core/src/model-profile/behaviors/prompt-verify-artifact-behavior.ts asking whether untrusted string lookup uses own properties and whether unknown input leaves state unchanged. Test it on a state machine and an unrelated interface/record lookup. No task names or expected fixture events in shared prompts.\n\n## Strategic experiment — not applied\n\nExtend code-validation guidance in packages/mcp/src/validate.ts to flag prototype-inclusive membership checks at string-input boundaries, and add property-based negative inputs to future separately versioned contract tests. The earlier interface-implementation advisory also found constructor/__proto__ lookup problems, supporting a reusable JavaScript boundary issue rather than one scenario.\n\n## Generalization and recommended next experiment\n\nRepetition 1 passed fixed 9.5 after four repair edits; repetition 2 passes 10.0 in one write with fewer calls. That efficiency improvement does not eliminate the newly confirmed unknown-event defect. Compare interface-implementation in the developer suite for the same inherited-key pattern, and pagination here for a narrow repair whose independently executed acceptance evidence matches the stated behavior.\n\nThe experiments above target reusable contract verification, state transitions, ownership or execution cost. Run each proposed change separately against the named task family and an unrelated task before recommending it; preserve baseline scores and avoid scenario-specific model hints. No prompt, tuning, source or gate change was applied.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/facts.json): pass, fixed 9.5, 19.05 min, 2.86 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/facts.json): fail, fixed 6.1, 2.28 min, 148.97 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-11-757Z-jc6t/facts.json): pass, fixed 10, 0.50 min, 139.69 tok/s.\n\nSame-campaign repetitions share the frozen identity; older runs with other models/releases are contextual only. No causal native-version improvement is claimed.\n\n## Strategic note\n\nScoped fixed acceptance is useful evidence; stronger user-facing claims require the separately described contract checks.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-engine-evidence.json, historical-comparison.json and the saved browser/contract checks where applicable.\n" }, { "cellId": "r2-saluki-cuda-core-symptom-debug", "baselineCellId": "r2-saluki-cuda-core-symptom-debug", "suite": "core", "scenario": "symptom-debug", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T11:31:24.505Z", "finishedAt": "2026-11-05T11:35:19.569Z", "durationMs": 235064, "success": true, "reason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 229311 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "symptom-debug", "bytes": 1306, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08912842107817544 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 229311 }, { "source": "facts", "path": "outcome.durationMs", "value": 235064 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 39, "peakRssMb": 9452, "peakCpuPercent": 1207, "firstAtMs": 1, "lastAtMs": 223965 }, "gpu": { "available": true, "sampleCount": 39, "peakUtilPercent": 98, "peakMemUsedMb": 10624, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 39, "peakUsedMb": 26090, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 5431, "totalOutputTokens": 1806, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:35:15.697Z" } }, "source": "engine-log" }, "derived": { "durationMs": 225404, "meanTokensPerSec": 8.92, "genTokensPerSec": 8.92, "genTokens": 1806, "promptTokensPerSec": 592, "peakRssMb": 9452, "peakGpuUtilPercent": 98, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:31:30.298Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:31:30.186Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "theo": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0552978515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0552978515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "durationMs": 235064, "timeoutMs": 2637363, "budgetUsedFraction": 0.08912842107817544 }, "timing": { "startedAt": "2026-11-05T11:31:24.505Z", "finishedAt": "2026-11-05T11:35:19.569Z", "timeToFirstArtifactMs": 229311, "timeToLastArtifactWriteMs": 229311, "timeToFirstTokenMs": 23614, "firstTurnTtftMs": 14136, "timeToFirstToolCallMs": 33383 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 4, "byTool": { "read_file": 3, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 7298, "filePath": "symptom-debug", "bytes": 1193, "score": 1, "scoreMax": 3, "signals": [ "accept-untouched" ], "failReason": "node accept.mjs still failing: CASE 3: expected \"a4\" got \"a5\" | CASE 4: expected [\"a1\",\"a2\",\"a3\",\"a4\",\"a5\",\"a6\",\"a7\",\"a8\",\"a9\",\"a10\"] got [\"a1\",\"a2\",\"a3\",\"a5\",\"a6\",\"a7\",\"a9\",\"a10\"] | CASE 6: expected [\"a1\",\"a2\",\"a3\",\"a4\",\"a5\",\"a6\",\"a7\",\"a8\",\"a9\",\"a10\"] got [\"a1\",\"a2\",\"a3\",\"a4\",\"a6\",\"a7\",\"a8\",\"a9\"]" }, { "atMs": 230891, "filePath": "symptom-debug", "bytes": 1306, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } ], "latest": { "filePath": "symptom-debug", "bytes": 1306, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T11:31:24.509Z] [trial] id=symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px scenario=symptom-debug model=underdog-saluki-27b-iq2-mix", "[2026-11-05T11:31:24.509Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "[2026-11-05T11:31:24.509Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-symptom-debug-r8GtUH", "[2026-11-05T11:31:24.509Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T11:31:24.509Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T11:31:24.511Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T11:31:24.515Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T11:31:24.521Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-symptom-debug-r8GtUH", "[2026-11-05T11:31:24.521Z] [trial] throughput-scaled ceiling: 20m → 44m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T11:31:30.246Z] [trial] daemon spawned pid=52220 port=https://127.0.0.1:49672", "[2026-11-05T11:31:31.134Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T11:31:31.152Z] [trial] meester=cosima", "[2026-11-05T11:31:31.152Z] [trial] running scenario setup hook", "[2026-11-05T11:31:31.573Z] [trial] scenario setup complete", "[2026-11-05T11:31:31.574Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T11:31:31.644Z] [poll] initial digests hard=3fb7e3bd2e7f89a3 soft=41fa9008505183d4", "[2026-11-05T11:31:31.923Z] [poll] hard-progress: 3fb7e3bd2e7f89a3 → 327bed494dfebec2 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/1193B)", "[2026-11-05T11:32:00.648Z] [poll] hard-progress: 327bed494dfebec2 → 0f59364e651a7958 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=25,stream=54) sniff=1/1193B)", "[2026-11-05T11:35:15.396Z] [poll] terminal: success=true reason=node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "[2026-11-05T11:35:15.726Z] [capture] dumped 1 session(s)", "[2026-11-05T11:35:15.731Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-05T11:35:15.804Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T11:35:19.568Z] [trial] daemon shut down", "[2026-11-05T11:35:19.577Z] [trial] PASS duration=235064ms reason=node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)" ] }, "metrics": { "process": { "sampleCount": 39, "peakRssMb": 9452, "peakCpuPercent": 1207, "firstAtMs": 1, "lastAtMs": 223965 }, "gpu": { "available": true, "sampleCount": 39, "peakUtilPercent": 98, "peakMemUsedMb": 10624, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 39, "peakUsedMb": 26090, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 5431, "totalOutputTokens": 1806, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T11:35:15.697Z" } }, "source": "engine-log" }, "derived": { "durationMs": 225404, "meanTokensPerSec": 8.92, "genTokensPerSec": 8.92, "genTokens": 1806, "promptTokensPerSec": 592, "peakRssMb": 9452, "peakGpuUtilPercent": 98, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T11:31:24.505Z", "finishedAt": "2026-11-05T11:35:19.569Z", "timeToFirstArtifactMs": 229311, "timeToLastArtifactWriteMs": 229311, "timeToFirstTokenMs": 23614, "firstTurnTtftMs": 14136, "timeToFirstToolCallMs": 33383 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:31:30.298Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:31:30.186Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "theo": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0552978515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0552978515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-symptom-debug", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "scenarioId": "symptom-debug", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/workspace/pagination-service/lib/paginate.mjs", "artifactBytes": 1306, "artifactSha256": "a33f753fa675cbf56b3c9066e16768fc555810fd79aa17af79b2d13a1b043c34", "axisSource": "evals/src/scenarios/symptom-debug.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. A targeted cursor correction makes the next-page token represent the last returned item, matching the existing resume-plus-one logic. All eight untouched acceptance cases pass. The file is concise and keeps input validation; broader malformed-cursor hardening was outside this repair brief.", "reviewBasis": "Read the full repaired module and the untouched eight-case acceptance script. The standard runner executed the real script and recorded ALL CASES PASS; no additional test run was needed.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/workspace/pagination-service/lib/paginate.mjs", "bytes": 1306, "sha256": "a33f753fa675cbf56b3c9066e16768fc555810fd79aa17af79b2d13a1b043c34" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/workspace/pagination-service/accept.mjs", "bytes": 1769, "sha256": "1a14b1c154b4faaca4d35d3d4124692e8bab41f43638161479c08a01fbf046a3" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:38.705Z", "reviewId": "049", "cellId": "r2-saluki-cuda-core-symptom-debug", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "scenarioId": "symptom-debug", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "baselineCellId": "r2-saluki-cuda-core-symptom-debug", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b766f1463f15108965891df6b51ad91997af81f94fc72e742da3aaecf69f1ade", "evidenceSha256": "40a55c50ba43c95a96cf2c9cebf65aecf56b6e52b5b6a952857ad77728d3e435", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/workspace/pagination-service/lib/paginate.mjs", "bytes": 1306, "sha256": "a33f753fa675cbf56b3c9066e16768fc555810fd79aa17af79b2d13a1b043c34", "shownBytes": 1306, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The shown paginate.mjs still looks inconsistent: decodeCursor+1 on resume paired with encodeCursor(nextIndex-1) is self-consistent, but the \"resume one past\" and cursor semantics may not match accept.mjs. The evidence is only a final file, with no accept.mjs or output.", "limitations": "accept.mjs, its failing output, the original file and the edit history are not shown, and no run outcome is shown. I can't tell whether the change fixes the cases or whether accept.mjs was left untouched. The run ended terminal, so I draw no success or failure conclusion.", "meanScore": null, "durationMs": 14285, "costUsd": 0.01198, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2169, "cache_read_input_tokens": 700, "output_tokens": 323, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2169, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 323, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2169, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2169 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 323, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2169, "webSearchRequests": 0, "costUSD": 0.01198, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2169, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - symptom-debug / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 3m 55s; budget used: 8.9%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 229311`; `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1306,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.08912842107817544`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: 229311`; `facts.outcome.durationMs: 235064` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9452 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 98% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10624 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26090 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.92 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 5431 + 1806 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 229311`\n- `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1306,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.08912842107817544`\n- `facts.toolUse.totalToolCalls: 4`\n- `facts.timing.timeToFirstArtifactMs: 229311`\n- `facts.outcome.durationMs: 235064`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 229311`; `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1306,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.08912842107817544`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: 229311`; `facts.outcome.durationMs: 235064`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is preserved.\n\nNo declared axes; prose review only. No numeric quality score was invented.\n\nNo declared axes. A targeted cursor correction makes the next-page token represent the last returned item, matching the existing resume-plus-one logic. All eight untouched acceptance cases pass. The file is concise and keeps input validation; broader malformed-cursor hardening was outside this repair brief.\n\nRead the full repaired module and the untouched eight-case acceptance script. The standard runner executed the real script and recorded ALL CASES PASS; no additional test run was needed.\n\n## Positives\n\n- One replacement after three reads repaired the boundary semantics while preserving the acceptance script (toolUse.byTool.replace_in_file: 1; read_file: 3; sniff.latest signals accept-untouched, paginate-changed, accept-passes).\n- The eight executable cases cover first/next pages, full traversal with two page sizes, uniqueness, oversized pages and empty input. The comment accurately explains why encodeCursor uses nextIndex-1 (outcome.reason; artifacts.otherFileCount; saved acceptance and module source).\n\n## Negatives\n\n- The 3m55s elapsed time still exceeds an interactive small-edit expectation even with only four tool calls; it is a performance observation, not a correctness penalty (outcome.durationMs: 235064; toolUse.totalToolCalls: 4; perf.derived.genTokens: 1806).\n- Tests establish the scoped pagination repair, not arbitrary cursor canonicalization or hostile input hardening. Those concerns must not be inferred solved from fixed 10.0 (sniff.latest 3/3; saved eight-case acceptance script).\n\n## Performance and GPU fit\n\nElapsed 3.92 minutes; 1806 generated tokens at 8.92 tok/s, prefill 592 tok/s. Peak VRAM 10624/12227 MiB; process-tree RSS 9452 MiB; whole-system RAM 26090/31865 MiB. Values come from perf.derived, perf.gpu, perf.process and perf.systemMemory; whole-system memory includes other applications.\n\nNative 0.1.57 CUDA, 65,536-token context, one slot, f16 KV, catalog reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. Shared GPU memory and paging were not measured. One Saluki launch; no explicit CUDA OOM, context overflow or unexpected native exit found. Keep performance, fixed capability and advisory quality distinct.\n\n## Tactical experiment — not applied\n\nOnly in a later isolated A/B, compare the current 4096 reasoning budget with 512 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json including the selected profile override, preserving output caps. Measure correct repairs, tokens and first-artifact time on pagination, state-machine work and an unrelated productivity task. This is a latency hypothesis, not a demonstrated fix.\n\n## Strategic experiment — not applied\n\nIn packages/inference/src/providers/llama-cpp/stdout-parser.ts and offload-planner.ts, emit structured per-turn placement/context/throughput records and correlate them with host load. Identical placement has shown roughly 2.9 and 8.9 tok/s in the two repetitions, so attributing all latency to CPU offload is unwarranted without more telemetry.\n\n## Generalization and recommended next experiment\n\nBoth repetitions score fixed 10.0 and repair the same cursor convention by different valid edits. Repetition 1 removed the resume increment; repetition 2 decrements the encoded next cursor. Elapsed time falls from 7m43s to 3m55s while output rises from 1151 to 1806 tokens and decode rises from 2.88 to 8.92 tok/s. Contrast migration and tankcombat: similar fixed passes there conceal broader semantic defects, so this successful small repair cannot establish diverse competence.\n\nThe experiments above target reusable contract verification, state transitions, ownership or execution cost. Run each proposed change separately against the named task family and an unrelated task before recommending it; preserve baseline scores and avoid scenario-specific model hints. No prompt, tuning, source or gate change was applied.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/facts.json): pass, fixed 10, 7.72 min, 2.88 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/facts.json): pass, fixed 10, 0.59 min, 118.38 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-30-48-132Z-5283/facts.json): pass, fixed 10, 0.85 min, 129.73 tok/s.\n\nSame-campaign repetitions share the frozen identity; older runs with other models/releases are contextual only. No causal native-version improvement is claimed.\n\n## Strategic note\n\nScoped fixed acceptance is useful evidence; stronger user-facing claims require the separately described contract checks.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-engine-evidence.json, historical-comparison.json and the saved browser/contract checks where applicable.\n" }, { "cellId": "r2-saluki-cuda-core-data-wrangle", "baselineCellId": "r2-saluki-cuda-core-data-wrangle", "suite": "core", "scenario": "data-wrangle", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T11:37:37.261Z", "finishedAt": "2026-11-05T13:23:11.060Z", "durationMs": 6333799, "success": false, "reason": "retry loop (fast-path): sniff \"data-wrangle:7:targetnone:frfe6uuy:rp0:rf0\" stuck for 11m — 3 re-writes (9 tool calls) without sniff movement. Artifact produced but never reached success.", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "data-wrangle", "score": 7, "bytes": 6986, "failReason": "records are not sorted by email ascending: \"tariq.aziz@example.com\" comes before \"sofia.keller@example.com\"" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "retry loop (fast-path): sniff \"data-wrangle:7:targetnone:frfe6uuy:rp0:rf0\" stuck for 11m — 3 re-writes (9 tool calls) without sniff movement. Artifact produced but never reached success." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1066310 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 6986, "score": 7, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count" ], "failReason": "records are not sorted by email ascending: \"tariq.aziz@example.com\" comes before \"sofia.keller@example.com\"" } } ] }, "quality": { "score": 6.4, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 63.6% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1066310 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 6986, "score": 7, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count" ], "failReason": "records are not sorted by email ascending: \"tariq.aziz@example.com\" comes before \"sofia.keller@example.com\"" } } ] }, "efficiency": { "score": 0, "ruleId": "E_BOTH_SEVERE", "summary": "The trial exhausted its budget while making at least thirty tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 43 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1066310 }, { "source": "facts", "path": "outcome.durationMs", "value": 6333799 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.3, "band": "framework-gap" }, "facts": { "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 1062, "peakRssMb": 15511, "peakCpuPercent": 1972.9, "firstAtMs": 1, "lastAtMs": 6322255 }, "gpu": { "available": true, "sampleCount": 1062, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1062, "peakUsedMb": 31312, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 114495, "totalOutputTokens": 48662, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 114133, "todayTokensCached": 71400, "todayTokensOut": 4529, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 114133, "totalTokensCached": 71400, "totalTokensOut": 4529, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 21686, "outputTokens": 843, "cost": 0, "durationMs": 166645, "outputTokensPerSec": 8.844103692263088, "at": "2026-11-05T13:23:04.388Z" }, "lastUpdated": "2026-11-05T13:23:04.388Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T13:23:06.403Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6323602, "meanTokensPerSec": 8.8, "genTokensPerSec": 8.8, "genTokens": 48662, "promptTokensPerSec": 383.28, "peakRssMb": 15511, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 114133, "output": 4529 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:37:42.906Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:37:42.807Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 12, "maxContextFill": 0.1914215087890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1914215087890625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "retry loop (fast-path): sniff \"data-wrangle:7:targetnone:frfe6uuy:rp0:rf0\" stuck for 11m — 3 re-writes (9 tool calls) without sniff movement. Artifact produced but never reached success.", "durationMs": 6333799, "timeoutMs": 5934066, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-05T11:37:37.261Z", "finishedAt": "2026-11-05T13:23:11.060Z", "timeToFirstArtifactMs": 1066310, "timeToLastArtifactWriteMs": 6327127, "timeToFirstTokenMs": 23764, "firstTurnTtftMs": 14437, "timeToFirstToolCallMs": 31334 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 43, "byTool": { "list_dir": 1, "read_file": 20, "stat": 4, "write_file": 6, "run_installed_script": 2, "replace_lines": 1, "replace_in_file": 8, "validate": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 7023, "filePath": "data-wrangle", "bytes": 0, "score": 0, "scoreMax": null, "signals": [], "failReason": null }, { "atMs": 1921152, "filePath": "data-wrangle", "bytes": 6728, "score": 6, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved" ], "failReason": "expected exactly 56 records after dedup, got 54" }, { "atMs": 4177953, "filePath": "data-wrangle", "bytes": 6986, "score": 7, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count" ], "failReason": "records are not sorted by email ascending: \"carmen.soto@example.com\" comes before \"cara.diaz@example.com\"" }, { "atMs": 5078921, "filePath": "data-wrangle", "bytes": 6986, "score": 7, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count" ], "failReason": "records are not sorted by email ascending: \"gleb.sokolov@example.com\" comes before \"gina.rossi@example.com\"" }, { "atMs": 5693269, "filePath": "data-wrangle", "bytes": 6986, "score": 7, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count" ], "failReason": "records are not sorted by email ascending: \"tariq.aziz@example.com\" comes before \"sofia.keller@example.com\"" } ], "latest": { "filePath": "data-wrangle", "bytes": 6986, "score": 7, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count" ], "failReason": "records are not sorted by email ascending: \"tariq.aziz@example.com\" comes before \"sofia.keller@example.com\"" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T11:37:37.265Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "[2026-11-05T11:37:37.265Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T11:37:37.265Z] [trial] id=data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q scenario=data-wrangle model=underdog-saluki-27b-iq2-mix", "[2026-11-05T11:37:37.265Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-data-wrangle-uBPxZa", "[2026-11-05T11:37:37.265Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T11:37:37.267Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T11:37:37.271Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T11:37:37.275Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-data-wrangle-uBPxZa", "[2026-11-05T11:37:37.275Z] [trial] throughput-scaled ceiling: 45m → 99m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T11:37:42.855Z] [trial] daemon spawned pid=9076 port=https://127.0.0.1:51967", "[2026-11-05T11:37:43.763Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T11:37:43.777Z] [trial] meester=ambrose", "[2026-11-05T11:37:43.777Z] [trial] running scenario setup hook", "[2026-11-05T11:37:44.198Z] [trial] scenario setup complete", "[2026-11-05T11:37:44.199Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T11:37:44.275Z] [poll] initial digests hard=e55730d148ef1bac soft=bc875ae0837fbae1", "[2026-11-05T11:37:44.324Z] [poll] hard-progress: e55730d148ef1bac → 46f10dc700379b77 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T11:38:13.441Z] [poll] hard-progress: 46f10dc700379b77 → 4e96ffdd30c94fa5 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=27,stream=58) sniff=0/0B)", "[2026-11-05T11:38:25.291Z] [poll] hard-progress: 4e96ffdd30c94fa5 → 525b96fc496776c3 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=44,stream=115) sniff=0/0B)", "[2026-11-05T11:46:14.461Z] [poll] hard-progress: 525b96fc496776c3 → 3101bce92200803f after 469s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=1432,stream=4216) sniff=0/0B)", "[2026-11-05T11:55:24.091Z] [poll] hard-progress: 3101bce92200803f → 94b35e96814be167 after 550s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=3040,stream=9623) sniff=0/0B)", "[2026-11-05T11:55:35.820Z] [poll] hard-progress: 94b35e96814be167 → 62da6bc9888bab7e after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=3058,stream=9679) sniff=0/0B)", "[2026-11-05T11:55:41.750Z] [poll] hard-progress: 62da6bc9888bab7e → 7a43cadc5a342af4 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=3069,stream=9716) sniff=0/0B)", "[2026-11-05T12:09:38.571Z] [poll] hard-progress: 7a43cadc5a342af4 → b840fa9d17a1f838 after 837s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=12,slot=5503,stream=19549) sniff=6/6728B)", "[2026-11-05T12:17:19.062Z] [poll] hard-progress: b840fa9d17a1f838 → dd425f3ded2f329a after 460s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=13,slot=6840,stream=23243) sniff=6/6728B)" ] }, "metrics": { "process": { "sampleCount": 1062, "peakRssMb": 15511, "peakCpuPercent": 1972.9, "firstAtMs": 1, "lastAtMs": 6322255 }, "gpu": { "available": true, "sampleCount": 1062, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1062, "peakUsedMb": 31312, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 114495, "totalOutputTokens": 48662, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 114133, "todayTokensCached": 71400, "todayTokensOut": 4529, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 114133, "totalTokensCached": 71400, "totalTokensOut": 4529, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 21686, "outputTokens": 843, "cost": 0, "durationMs": 166645, "outputTokensPerSec": 8.844103692263088, "at": "2026-11-05T13:23:04.388Z" }, "lastUpdated": "2026-11-05T13:23:04.388Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T13:23:06.403Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6323602, "meanTokensPerSec": 8.8, "genTokensPerSec": 8.8, "genTokens": 48662, "promptTokensPerSec": 383.28, "peakRssMb": 15511, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 114133, "output": 4529 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T11:37:37.261Z", "finishedAt": "2026-11-05T13:23:11.060Z", "timeToFirstArtifactMs": 1066310, "timeToLastArtifactWriteMs": 6327127, "timeToFirstTokenMs": 23764, "firstTurnTtftMs": 14437, "timeToFirstToolCallMs": 31334 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T11:37:42.906Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T11:37:42.807Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 12, "maxContextFill": 0.1914215087890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1914215087890625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-data-wrangle", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "scenarioId": "data-wrangle", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/workspace/customer-data-cleanup/out/customers.json", "artifactBytes": 6986, "artifactSha256": "11bcc09a93017c22a2bc704f81585d33bc222c2aae6e19b48b52f98ec0342f77", "axisSource": "evals/src/scenarios/data-wrangle.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. All 56 deduplicated source records, ids, names and dates are correct, but Tariq remains before Sofia/Stefan rather than in email order. The standard runner ended after repeated edits stalled at 7/11. This is a partial data deliverable, not a load, OOM or context failure.", "reviewBasis": "Read the full final JSON and compared its complete record set with the independently source-verified 12:55 snapshot. Inspected all failed execution-tool calls, validation output, edits and final fixed facts. No candidate output was repaired or rerun.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/workspace/customer-data-cleanup/out/customers.json", "bytes": 6986, "sha256": "11bcc09a93017c22a2bc704f81585d33bc222c2aae6e19b48b52f98ec0342f77" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:45.866Z", "reviewId": "050", "cellId": "r2-saluki-cuda-core-data-wrangle", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "scenarioId": "data-wrangle", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "baselineCellId": "r2-saluki-cuda-core-data-wrangle", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "54fb5989362d42fe3caaafdb7225e2a23a2a37a6dd8a6c35a0b51850e2a2a385", "evidenceSha256": "e6639124acba222ceae2563cd01b5796b9c375f418c96356f961487e18ef47b1", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/workspace/customer-data-cleanup/out/customers.json", "bytes": 6986, "sha256": "11bcc09a93017c22a2bc704f81585d33bc222c2aae6e19b48b52f98ec0342f77", "shownBytes": 6986, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q\\workspace\\customer-data-cleanup\\data\\raw\\customers_a.csv", "bytes": 1045, "sha256": "bc92617373571bdf0bef1c29ee05aad44a7a68274c0f7334b1ebe8aa5eedfe54", "shownBytes": 1045, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q\\workspace\\customer-data-cleanup\\data\\raw\\customers_b.csv", "bytes": 1087, "sha256": "ff167222077dd5a971ee88d6ab656d98b2b2051e84ceb223bcc893aab9186b09", "shownBytes": 1087, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q\\workspace\\customer-data-cleanup\\data\\raw\\legacy_export.csv", "bytes": 1148, "sha256": "90e6abcfcf404de96b3ac6385e3ddc3265125552377f3c3428d5636d310f9afa", "shownBytes": 1148, "truncated": false } ], "scoreAxes": {}, "justification": "56 records, correct for 60 rows minus 4 duplicates. Newest-date dedup is right for alice, robert, omar and zoe. Dates, quoted names and ids look right. Defect: tariq.aziz appears before sofia.keller and stefan.beck, so the email sort is wrong.", "limitations": "Source-only review; the validator was not run, and I did not check every date conversion. I did not verify the file has no BOM. I saw no transcript, so I don't know whether the agent noticed the sort error.", "meanScore": null, "durationMs": 18594, "costUsd": 0.046384, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 9140, "cache_read_input_tokens": 700, "output_tokens": 975, "output_tokens_details": { "thinking_tokens": 688 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 9140, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 975, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 9140, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9140 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 975, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 9140, "webSearchRequests": 0, "costUSD": 0.046384, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 688, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 9140, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - data-wrangle / underdog-saluki-27b-iq2-mix\n\n**Composite: 4.3 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 105m 34s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"data-wrangle:7:targetnone:frfe6uuy:rp0:rf0\\\" stuck for 11m — 3 re-writes (9 tool calls) without sniff movement. Artifact produced but never reac...`; `facts.timing.timeToFirstArtifactMs: 1066310`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":7,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",\"...` |\n| Output quality (25%) | 6.4 / 10 | The latest scenario gate reached 63.6% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1066310`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":7,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",\"...` |\n| Process efficiency (20%) | 0 / 10 | The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 43`; `facts.timing.timeToFirstArtifactMs: 1066310`; `facts.outcome.durationMs: 6333799` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 6.4 + 0.2 * 0 + 0.15 * 10 = **4.3**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15511 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31312 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.8 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 114495 + 48662 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 19% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"data-wrangle:7:targetnone:frfe6uuy:rp0:rf0\\\" stuck for 11m — 3 re-writes (9 tool calls) without sniff movement. Artifact produced but never reac...`\n- `facts.timing.timeToFirstArtifactMs: 1066310`\n- `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":7,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",\"...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 63.6% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 1066310`\n- `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":7,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",\"...`\n\n### Process efficiency - E_BOTH_SEVERE\n\nThe trial exhausted its budget while making at least thirty tool calls.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 43`\n- `facts.timing.timeToFirstArtifactMs: 1066310`\n- `facts.outcome.durationMs: 6333799`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"data-wrangle:7:targetnone:frfe6uuy:rp0:rf0\\\" stuck for 11m — 3 re-writes (9 tool calls) without sniff movement. Artifact produced but never reac...`; `facts.timing.timeToFirstArtifactMs: 1066310`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":7,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",\"...`\n- Quality: The latest scenario gate reached 63.6% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1066310`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":7,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",\"...`\n- Efficiency: The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 43`; `facts.timing.timeToFirstArtifactMs: 1066310`; `facts.outcome.durationMs: 6333799`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **fail, 4.3/10** is preserved. This scenario has no declared axes; the advisory review is prose only.\n\nNo declared axes. All 56 deduplicated source records, ids, names and dates are correct, but Tariq remains before Sofia/Stefan rather than in email order. The standard runner ended after repeated edits stalled at 7/11. This is a partial data deliverable, not a load, OOM or context failure.\n\nRead the full final JSON and compared its complete record set with the independently source-verified 12:55 snapshot. Inspected all failed execution-tool calls, validation output, edits and final fixed facts. No candidate output was repaired or rerun.\n\n## Positives\n\n- A real 56-row dataset was delivered, with all retained records matching the newest raw source row for each normalized email. Independent comparison finds no missing or unexpected emails or field mismatches (artifacts.otherFileCount; sniff.latest row-count/source-ids-preserved; advisory-data-audit.json and 12:55 source audit).\n- The model corrected five of the six adjacent order inversions seen at the prior checkpoint. It preserved raw identifiers, quoted names, normalized dates and the newest duplicate selections (sniff.latest 7/11; final output versus prior reviewed snapshot).\n\n## Negatives\n\n- One remaining adjacent inversion puts tariq.aziz before sofia.keller; Tariq also needs to follow Stefan in the sorted sequence. The standard runner ended model-stuck after three rewrites without sniff movement, not a hardware/infrastructure failure (outcome.failureMode; outcome.reason; sniff.latest.failReason).\n- Two run_installed_script calls failed, with no successful cleaner/checker execution recorded. One error suggests run_nodejs_script with the entire string \"node tools/clean_customers.mjs\" as its path; both permit direct hand-written output if execution is unavailable. The later validate PASS covers only nonempty JSON and parsing, not sort or source semantics (toolUse.byTool.run_installed_script: 2; validate: 1; recorded tool results).\n- The run used 43 calls and 48,662 generated tokens over 105m34s, repeatedly editing data instead of completing a deterministic transform. The earlier helper snapshots also include headers as data. Separate those code defects from the tool-dispatch failures; neither was repaired by this review (toolUse; perf.derived.genTokens; outcome.durationMs: 6333799; saved helper snapshots).\n\n## Performance and GPU fit\n\nElapsed 105.56 minutes; 48662 generated tokens at 8.8 tok/s, prefill 383.28 tok/s. Peak VRAM 10642/12227 MiB; process-tree RSS 15511 MiB; whole-system RAM 31312/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications.\n\nNative 0.1.57 CUDA; one launch, 65,536-token context, one slot, f16 KV and reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. No explicit OOM, context overflow or unexpected native exit was found. Shared GPU memory/paging is unmeasured. Performance stays separate from capability and advisory quality.\n\n## Tactical experiment — not applied\n\nIn packages/core/src/model-profile/behaviors/prompt-derive-by-execution.ts, extend the existing roster-aware guidance to require one available execution route plus validator output before treating a derived dataset as complete. Do not redirect to unavailable tools or hand-retouch rows after execution failure. A/B on customer normalization and an unrelated table aggregation; measure exact source fidelity, successful execution and tool cost.\n\n## Strategic experiment — not applied\n\nIn packages/mcp/src/installed-script-miss.ts, distinguish shell-command text from a filesystem path and generate a valid structured next action only when that tool is available. In packages/mcp/src/validate.ts, label JSON parse-only validation as syntactic and expose the missing semantic contract explicitly. Reconcile the actual execution-tool roster for data tasks instead of altering this anchored dataset or relaxing sorting.\n\n## Generalization and next experiment\n\nRepetition 1 failed fixed 3.0 with no JSON; repetition 2 improves to fixed 4.3 with correct records but unfinished sorting. Both expose execution-route trouble. The successful pagination repair demonstrates that small direct edits can work, while precision datasets need reliable deterministic execution. No failed capability cell was retried.\n\nTest each proposed change separately against this task family and an unrelated task before treating it as an improvement. No source, prompt, tuning, gate, model or campaign state was changed, and no capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/facts.json): fail, fixed 3, 56.38 min, 2.83 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/facts.json): fail, fixed 3.4, 9.26 min, 141.01 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-36-45-341Z-u18k/facts.json): fail, fixed 3.4, 12.11 min, 135.74 tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are historical context; no causal native-version or tuning improvement is inferred.\n\n## Strategic note\n\nThe repeated execution-route failure remains actionable even though record fidelity improved. A parse-only validator must not be mistaken for the full data contract.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, historical-comparison.json, recording/chat-events.jsonl and the saved source/output files.\n" }, { "cellId": "r2-saluki-cuda-core-incident-postmortem", "baselineCellId": "r2-saluki-cuda-core-incident-postmortem", "suite": "core", "scenario": "incident-postmortem", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T13:26:14.308Z", "finishedAt": "2026-11-05T13:41:18.352Z", "durationMs": 904044, "success": true, "reason": "all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 892906 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "incident-postmortem", "bytes": 8061, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.17139168392665022 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 892906 }, { "source": "facts", "path": "outcome.durationMs", "value": 904044 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 151, "peakRssMb": 11253, "peakCpuPercent": 1201.4, "firstAtMs": 1, "lastAtMs": 889963 }, "gpu": { "available": true, "sampleCount": 151, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 151, "peakUsedMb": 28470, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 30374, "totalOutputTokens": 6839, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T13:41:12.051Z" } }, "source": "engine-log" }, "derived": { "durationMs": 891501, "meanTokensPerSec": 8.46, "genTokensPerSec": 8.46, "genTokens": 6839, "promptTokensPerSec": 475.95, "peakRssMb": 11253, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T13:26:20.114Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T13:26:20.005Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "jordan": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.14569091796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.14569091796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "durationMs": 904044, "timeoutMs": 5274725, "budgetUsedFraction": 0.17139168392665022 }, "timing": { "startedAt": "2026-11-05T13:26:14.308Z", "finishedAt": "2026-11-05T13:41:18.352Z", "timeToFirstArtifactMs": 892906, "timeToLastArtifactWriteMs": 892906, "timeToFirstTokenMs": 35069, "firstTurnTtftMs": 25045, "timeToFirstToolCallMs": 58489 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 8, "byTool": { "read_file": 6, "write_file": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 897247, "filePath": "incident-postmortem", "bytes": 8061, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } ], "latest": { "filePath": "incident-postmortem", "bytes": 8061, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T13:26:14.311Z] [trial] id=incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do scenario=incident-postmortem model=underdog-saluki-27b-iq2-mix", "[2026-11-05T13:26:14.311Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "[2026-11-05T13:26:14.311Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T13:26:14.311Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-incident-postmortem-w0ZZkN", "[2026-11-05T13:26:14.311Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T13:26:14.314Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T13:26:14.318Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T13:26:14.324Z] [trial] throughput-scaled ceiling: 40m → 88m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T13:26:14.324Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-incident-postmortem-w0ZZkN", "[2026-11-05T13:26:20.060Z] [trial] daemon spawned pid=11948 port=https://127.0.0.1:58695", "[2026-11-05T13:26:21.418Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T13:26:21.434Z] [trial] meester=cassian", "[2026-11-05T13:26:21.435Z] [trial] running scenario setup hook", "[2026-11-05T13:26:21.856Z] [trial] scenario setup complete", "[2026-11-05T13:26:21.858Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T13:26:21.932Z] [poll] initial digests hard=3f08ee55f26ee1a7 soft=ca68cf96600922d4", "[2026-11-05T13:26:22.009Z] [poll] hard-progress: 3f08ee55f26ee1a7 → a0cb48d7b090a858 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T13:27:14.459Z] [poll] hard-progress: a0cb48d7b090a858 → 1325e40edbb020c9 after 52s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=55,stream=145) sniff=0/0B)", "[2026-11-05T13:35:03.431Z] [poll] hard-progress: 1325e40edbb020c9 → f83dc501888013db after 469s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=1349,stream=6488) sniff=0/0B)", "[2026-11-05T13:35:32.975Z] [poll] hard-progress: f83dc501888013db → 45d8611558d97182 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=1363,stream=6508) sniff=0/0B)", "[2026-11-05T13:41:11.555Z] [poll] terminal: success=true reason=all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "[2026-11-05T13:41:12.116Z] [capture] dumped 1 session(s)", "[2026-11-05T13:41:12.120Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-05T13:41:12.193Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T13:41:18.350Z] [trial] daemon shut down" ] }, "metrics": { "process": { "sampleCount": 151, "peakRssMb": 11253, "peakCpuPercent": 1201.4, "firstAtMs": 1, "lastAtMs": 889963 }, "gpu": { "available": true, "sampleCount": 151, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 151, "peakUsedMb": 28470, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 30374, "totalOutputTokens": 6839, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T13:41:12.051Z" } }, "source": "engine-log" }, "derived": { "durationMs": 891501, "meanTokensPerSec": 8.46, "genTokensPerSec": 8.46, "genTokens": 6839, "promptTokensPerSec": 475.95, "peakRssMb": 11253, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T13:26:14.308Z", "finishedAt": "2026-11-05T13:41:18.352Z", "timeToFirstArtifactMs": 892906, "timeToLastArtifactWriteMs": 892906, "timeToFirstTokenMs": 35069, "firstTurnTtftMs": 25045, "timeToFirstToolCallMs": 58489 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T13:26:20.114Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T13:26:20.005Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "jordan": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.14569091796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.14569091796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-incident-postmortem", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "scenarioId": "incident-postmortem", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/workspace/checkout-incident-postmortem/postmortem.md", "artifactBytes": 8111, "artifactSha256": "c667729420b47878078574e88bcf84ac16dadf728a74c3087dfe9dc3fb23c7f1", "axisSource": "evals/src/scenarios/incident-postmortem.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The report covers the required sections, core timeout/pool mechanism, evidence citations and concrete actions. It explicitly leaves data-loss/security impact unknown. It overstates absent monitoring and merge-delay causality, and repeats inconsistent incident timing without reconciling the source conflict.", "reviewBasis": "Read the full report and all five immutable evidence files. Compared metrics and timestamps, causal claims, action tables and both write attempts. The fixed pass is preserved; these are advisory grounding qualifications.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/workspace/checkout-incident-postmortem/postmortem.md", "bytes": 8111, "sha256": "c667729420b47878078574e88bcf84ac16dadf728a74c3087dfe9dc3fb23c7f1" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:45.356Z", "reviewId": "051", "cellId": "r2-saluki-cuda-core-incident-postmortem", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "scenarioId": "incident-postmortem", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "baselineCellId": "r2-saluki-cuda-core-incident-postmortem", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "09e3a28d8b59268ba8fc39797eb41d4315e74217f51cf6db1f538b1c38861b18", "evidenceSha256": "95554f8536bbc8f245665026ff3be84aaeac9307648f5c7981c9ad52f0b08ab0", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/workspace/checkout-incident-postmortem/postmortem.md", "bytes": 8111, "sha256": "c667729420b47878078574e88bcf84ac16dadf728a74c3087dfe9dc3fb23c7f1", "shownBytes": 8111, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. Qualitatively, the postmortem has all eight sections in order, blame-free tone, 5 cited files, many HH:MM times, and an action table with concrete owners. It treats data loss/security as unknown. Some details, such as the claimed 6-hour merge gap, couldn't be verified here.", "limitations": "Evidence files weren't shown, so claims (timestamps, metrics, review details, the \"~13 minutes\" figure) are unverified. Size is about 8 KB, within range. Some Impact/Timeline inconsistencies are possible, such as 14:53 vs 14:54 recovery. Immutability of the evidence files and the tool-call sequence cannot be confirmed.", "meanScore": null, "durationMs": 14935, "costUsd": 0.025384, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5445, "cache_read_input_tokens": 700, "output_tokens": 353, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5445, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 353, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 5445, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5445 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 353, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 5445, "webSearchRequests": 0, "costUSD": 0.025384, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5445, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - incident-postmortem / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 15m 4s; budget used: 17.1%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 892906`; `facts.sniff.latest: {\"filePath\":\"incident-postmortem\",\"bytes\":8061,\"score\":9,\"scoreMax\":9,\"signals\":[\"file-present\",\"all-sections\",\"summary-concise\",\"evidence-citations\",\"timestamp-citations\",\"grou...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.17139168392665022`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 892906`; `facts.outcome.durationMs: 904044` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11253 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28470 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.46 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 30374 + 6839 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 892906`\n- `facts.sniff.latest: {\"filePath\":\"incident-postmortem\",\"bytes\":8061,\"score\":9,\"scoreMax\":9,\"signals\":[\"file-present\",\"all-sections\",\"summary-concise\",\"evidence-citations\",\"timestamp-citations\",\"grou...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.17139168392665022`\n- `facts.toolUse.totalToolCalls: 8`\n- `facts.timing.timeToFirstArtifactMs: 892906`\n- `facts.outcome.durationMs: 904044`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 892906`; `facts.sniff.latest: {\"filePath\":\"incident-postmortem\",\"bytes\":8061,\"score\":9,\"scoreMax\":9,\"signals\":[\"file-present\",\"all-sections\",\"summary-concise\",\"evidence-citations\",\"timestamp-citations\",\"grou...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.17139168392665022`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 892906`; `facts.outcome.durationMs: 904044`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is preserved. This scenario has no declared axes; the advisory review is prose only.\n\nNo declared axes. The report covers the required sections, core timeout/pool mechanism, evidence citations and concrete actions. It explicitly leaves data-loss/security impact unknown. It overstates absent monitoring and merge-delay causality, and repeats inconsistent incident timing without reconciling the source conflict.\n\nRead the full report and all five immutable evidence files. Compared metrics and timestamps, causal claims, action tables and both write attempts. The fixed pass is preserved; these are advisory grounding qualifications.\n\n## Positives\n\n- All eight hard checks and the blame-language advisory pass. The report explains the 800ms-to-8000ms change, saturated pool, 504s and successful revert with citations to the diff, metrics, deploy log and on-call conversation (outcome.reason; sniff.latest signals; artifacts.otherFileCount).\n- The report explicitly treats data loss/security effects as unknown, and provides owned, dated follow-up actions. After a grounding rejection it reread evidence and removed the unsupported approximately-19-minute saturation duration (toolUse.byTool.write_file: 2, read_file: 6; first rejected write and final accepted report).\n\n## Negatives\n\n- The report says saturation was not monitored/alerted on and treats a six-hour merge delay as increasing blast radius. The pack shows one error-rate page and an elapsed merge gap, but neither absence of all other alerts nor that causal effect is established (artifacts.otherFileCount; final report versus five source files).\n- The summary associates 99% saturation with the first minute/14:33, but metrics show 97% at 14:33 and 99% at 14:34. More materially, the timeline says 14:58 resolved after a 10-minute monitoring window, while the chat starts that window at 14:58 and closes at 15:08. The report repeats the former without noting the conflict; it does correctly use the deployment log start of 14:46:30 instead of timeline.md’s 14:51 (sniff.latest timestamp-citations/grounded-core-facts; inspected metric and timeline rows).\n- The grounding guard caught one unsupported duration but accepted unsupported absence/causality claims later. Citation presence is not sufficient support for a claim (recording/chat-events.jsonl rejected 13:35 write; accepted 13:41 write; final text).\n\n## Performance and GPU fit\n\nElapsed 15.07 minutes; 6839 generated tokens at 8.46 tok/s, prefill 475.95 tok/s. Peak VRAM 10630/12227 MiB; process-tree RSS 11253 MiB; whole-system RAM 28470/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications.\n\nNative 0.1.57 CUDA; one launch, 65,536-token context, one slot, f16 KV and reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. No explicit OOM, context overflow or unexpected native exit was found. Shared GPU memory/paging is unmeasured. Performance stays separate from capability and advisory quality.\n\n## Tactical experiment — not applied\n\nIn a proposed generic packages/core/src/model-profile/behaviors/prompt-evidence-conflicts.ts block, require explicit attribution of conflicting timestamps and distinguish \"not shown in this evidence\" from \"did not happen.\" Label causal hypotheses and proposed actions. Gate on source-grounded writing tasks and compare an incident report with an unrelated policy/conflict synthesis.\n\n## Strategic experiment — not applied\n\nExtend the service factual-grounding pipeline reached from packages/service/src/chat/manager.ts to review absence and causal claims as well as unsupported numerals. Preserve competing source facts and record why one is preferred. A later separately versioned evaluator can check conflict handling, while this campaign’s fixed gate remains unchanged.\n\n## Generalization and next experiment\n\nRepetition 1 failed fixed 3.5 without a report; repetition 2 passes 10.0 in 15m04s. This establishes improved delivery in the second trial, not perfect evidence interpretation. Compare the runbook: precise observed-versus-expected reporting succeeds there, whereas the incident narrative adds unsupported generalizations beyond the evidence.\n\nTest each proposed change separately against this task family and an unrelated task before treating it as an improvement. No source, prompt, tuning, gate, model or campaign state was changed, and no capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/facts.json): fail, fixed 3.5, 34.06 min, 2.76 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/facts.json): pass, fixed 10, 2.00 min, 101.82 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T07-59-49-489Z-sao6/facts.json): pass, fixed 10, 1.89 min, 106.14 tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are historical context; no causal native-version or tuning improvement is inferred.\n\n## Strategic note\n\nReport delivery improved from absent to complete; source-conflict and causal-claim handling still need review.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, historical-comparison.json, recording/chat-events.jsonl and the saved source/output files.\n" }, { "cellId": "r2-saluki-cuda-core-ops-runbook-anomaly", "baselineCellId": "r2-saluki-cuda-core-ops-runbook-anomaly", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T13:43:51.253Z", "finishedAt": "2026-11-05T13:49:45.461Z", "durationMs": 354208, "success": true, "reason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 130844 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "ops-runbook-anomaly", "bytes": 1328, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07674506027124497 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 15 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 130844 }, { "source": "facts", "path": "outcome.durationMs", "value": 354208 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 59, "peakRssMb": 14687, "peakCpuPercent": 1198.3, "firstAtMs": 0, "lastAtMs": 343337 }, "gpu": { "available": true, "sampleCount": 59, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 59, "peakUsedMb": 31080, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 48898, "totalOutputTokens": 1722, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18512, "todayTokensCached": 12381, "todayTokensOut": 477, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18512, "totalTokensCached": 12381, "totalTokensOut": 477, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5877, "outputTokens": 252, "cost": 0, "durationMs": 147881, "outputTokensPerSec": 8.961395664184016, "at": "2026-11-05T13:49:37.250Z" }, "lastUpdated": "2026-11-05T13:49:37.250Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T13:49:41.593Z" } }, "source": "engine-log" }, "derived": { "durationMs": 344650, "meanTokensPerSec": 9, "genTokensPerSec": 9, "genTokens": 1722, "promptTokensPerSec": 410.13, "peakRssMb": 14687, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 18512, "output": 477 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T13:43:56.957Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T13:43:56.859Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "onno": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.0800933837890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0800933837890625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "durationMs": 354208, "timeoutMs": 4615385, "budgetUsedFraction": 0.07674506027124497 }, "timing": { "startedAt": "2026-11-05T13:43:51.253Z", "finishedAt": "2026-11-05T13:49:45.461Z", "timeToFirstArtifactMs": 130844, "timeToLastArtifactWriteMs": 345997, "timeToFirstTokenMs": 37353, "firstTurnTtftMs": 26292, "timeToFirstToolCallMs": 43604 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 15, "byTool": { "read_file": 12, "write_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 10 }, "sniff": { "progression": [ { "atMs": 131549, "filePath": "ops-runbook-anomaly", "bytes": 451, "score": 5, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion" ], "failReason": "halt-report.md does not exist — re-read runbook.md and state/backup.json, then report the failing step/source plus the expected and observed status/age fields without executing later steps" }, { "atMs": 167002, "filePath": "ops-runbook-anomaly", "bytes": 1114, "score": 6, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded" ], "failReason": null }, { "atMs": 196362, "filePath": "ops-runbook-anomaly", "bytes": 1114, "score": 6, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded" ], "failReason": "source-read provenance cannot identify committed final recording call(s): STEP 4 recording" }, { "atMs": 202411, "filePath": "ops-runbook-anomaly", "bytes": 1114, "score": 6, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded" ], "failReason": null }, { "atMs": 349814, "filePath": "ops-runbook-anomaly", "bytes": 1328, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } ], "latest": { "filePath": "ops-runbook-anomaly", "bytes": 1328, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T13:43:51.256Z] [trial] id=ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3 scenario=ops-runbook-anomaly model=underdog-saluki-27b-iq2-mix", "[2026-11-05T13:43:51.256Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-ops-runbook-anomaly-NNfrWZ", "[2026-11-05T13:43:51.256Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "[2026-11-05T13:43:51.256Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T13:43:51.256Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T13:43:51.259Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T13:43:51.262Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T13:43:51.268Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-ops-runbook-anomaly-NNfrWZ", "[2026-11-05T13:43:51.268Z] [trial] throughput-scaled ceiling: 35m → 77m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T13:43:56.890Z] [trial] daemon spawned pid=49748 port=https://127.0.0.1:56006", "[2026-11-05T13:43:57.765Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T13:43:57.784Z] [trial] running scenario setup hook", "[2026-11-05T13:43:57.784Z] [trial] meester=zara", "[2026-11-05T13:43:58.213Z] [trial] scenario setup complete", "[2026-11-05T13:43:58.215Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T13:43:58.289Z] [poll] initial digests hard=ed96c7a883a90e36 soft=b798f5ce4ec982ae", "[2026-11-05T13:43:58.367Z] [poll] hard-progress: ed96c7a883a90e36 → c30601ae7ade98ed after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T13:44:40.189Z] [poll] hard-progress: c30601ae7ade98ed → 425458b87b1e02da after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=36,stream=71) sniff=0/0B)", "[2026-11-05T13:44:51.979Z] [poll] hard-progress: 425458b87b1e02da → 85acda1ebc2a59ee after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=61,stream=145) sniff=0/0B)", "[2026-11-05T13:45:03.831Z] [poll] hard-progress: 85acda1ebc2a59ee → 7135b086e61a2481 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=90,stream=229) sniff=0/0B)", "[2026-11-05T13:45:15.668Z] [poll] hard-progress: 7135b086e61a2481 → 5b018be36078f28e after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=113,stream=298) sniff=0/0B)", "[2026-11-05T13:45:21.569Z] [poll] hard-progress: 5b018be36078f28e → 66706239f36cb128 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=126,stream=337) sniff=0/0B)", "[2026-11-05T13:46:02.881Z] [poll] hard-progress: 66706239f36cb128 → 77d4cb07e552db08 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=239,stream=825) sniff=5/451B)", "[2026-11-05T13:46:38.337Z] [poll] hard-progress: 77d4cb07e552db08 → 3bfdccdd098d8479 after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=331,stream=1270) sniff=6/1114B)", "[2026-11-05T13:47:25.595Z] [poll] hard-progress: 3bfdccdd098d8479 → 857bd613ae906f8e after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=9,slot=419,stream=1504) sniff=6/1114B)" ] }, "metrics": { "process": { "sampleCount": 59, "peakRssMb": 14687, "peakCpuPercent": 1198.3, "firstAtMs": 0, "lastAtMs": 343337 }, "gpu": { "available": true, "sampleCount": 59, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 59, "peakUsedMb": 31080, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 48898, "totalOutputTokens": 1722, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18512, "todayTokensCached": 12381, "todayTokensOut": 477, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18512, "totalTokensCached": 12381, "totalTokensOut": 477, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5877, "outputTokens": 252, "cost": 0, "durationMs": 147881, "outputTokensPerSec": 8.961395664184016, "at": "2026-11-05T13:49:37.250Z" }, "lastUpdated": "2026-11-05T13:49:37.250Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T13:49:41.593Z" } }, "source": "engine-log" }, "derived": { "durationMs": 344650, "meanTokensPerSec": 9, "genTokensPerSec": 9, "genTokens": 1722, "promptTokensPerSec": 410.13, "peakRssMb": 14687, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 18512, "output": 477 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T13:43:51.253Z", "finishedAt": "2026-11-05T13:49:45.461Z", "timeToFirstArtifactMs": 130844, "timeToLastArtifactWriteMs": 345997, "timeToFirstTokenMs": 37353, "firstTurnTtftMs": 26292, "timeToFirstToolCallMs": 43604 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T13:43:56.957Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T13:43:56.859Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "onno": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.0800933837890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0800933837890625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "scenarioId": "ops-runbook-anomaly", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/workspace/quarterly-maintenance-window/runlog.md", "artifactBytes": 667, "artifactSha256": "1655b285ef26726484acb4436918fd5224d6e5c2fe0627a6199023173e03d306", "axisSource": "evals/src/scenarios/ops-runbook.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The log records 14 services, the exact checksum and config version, then stops at a stale 71-hour backup. The halt report accurately contrasts fresh/<24h with stale/71h. Forbidden maintenance/decommission outputs are absent. The scoped procedure is correctly completed by halting.", "reviewBasis": "Read both delivered documents, the runbook, all four state inputs and expected checksum. Inspected saved output inventory and the source-read/write trace; no numeric axes were invented.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/workspace/quarterly-maintenance-window/runlog.md", "bytes": 667, "sha256": "1655b285ef26726484acb4436918fd5224d6e5c2fe0627a6199023173e03d306" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/workspace/quarterly-maintenance-window/halt-report.md", "bytes": 669, "sha256": "4539f21a701c016b89078a18c64532e9afdcafcd940c893b0765810344349c3e" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:53.579Z", "reviewId": "052", "cellId": "r2-saluki-cuda-core-ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "scenarioId": "ops-runbook-anomaly", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "baselineCellId": "r2-saluki-cuda-core-ops-runbook-anomaly", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "d4415f3e245a33f1890cd576d091f60ad947074d9b9d18de50b795d70384bfdb", "evidenceSha256": "8d043ca5949a408cd7cfed5fd24d18d5241972e114faeaaeadcec0d630a15515", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/workspace/quarterly-maintenance-window/runlog.md", "bytes": 667, "sha256": "1655b285ef26726484acb4436918fd5224d6e5c2fe0627a6199023173e03d306", "shownBytes": 667, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The runlog records Steps 1–3 with verbatim values and a Step 4 failure (backup stale, age 71h vs fresh/<24h), and states steps 5–6 were not run. That matches the halt rule. Only runlog.md is shown, so halt-report.md is unverified.", "limitations": "Only runlog.md was provided. halt-report.md, the tool-call order, whether state files were read first, and whether steps 5–6 outputs were absent can't be verified. Step 1's recorded \"14 services\" can't be checked against the source file.", "meanScore": null, "durationMs": 14833, "costUsd": 0.010683999999999999, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1890, "cache_read_input_tokens": 700, "output_tokens": 305, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1890, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 305, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1890, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1890 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 305, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1890, "webSearchRequests": 0, "costUSD": 0.010683999999999999, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1890, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - ops-runbook-anomaly / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 5m 54s; budget used: 7.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 130844`; `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":1328,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.07674506027124497`; `facts.toolUse.totalToolCalls: 15`; `facts.timing.timeToFirstArtifactMs: 130844`; `facts.outcome.durationMs: 354208` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14687 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10636 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31080 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 48898 + 1722 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 8% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 130844`\n- `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":1328,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.07674506027124497`\n- `facts.toolUse.totalToolCalls: 15`\n- `facts.timing.timeToFirstArtifactMs: 130844`\n- `facts.outcome.durationMs: 354208`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 130844`; `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":1328,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.07674506027124497`; `facts.toolUse.totalToolCalls: 15`; `facts.timing.timeToFirstArtifactMs: 130844`; `facts.outcome.durationMs: 354208`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 9.5/10** is preserved. This scenario has no declared axes; the advisory review is prose only.\n\nNo declared axes. The log records 14 services, the exact checksum and config version, then stops at a stale 71-hour backup. The halt report accurately contrasts fresh/<24h with stale/71h. Forbidden maintenance/decommission outputs are absent. The scoped procedure is correctly completed by halting.\n\nRead both delivered documents, the runbook, all four state inputs and expected checksum. Inspected saved output inventory and the source-read/write trace; no numeric axes were invented.\n\n## Positives\n\n- Steps 1–3 record the exact observed values, and step 4 correctly identifies both freshness and age violations. The halt report names expected and actual values (sniff.latest signals step1-recorded, step2-recorded, step3-recorded, halt-report-grounded; both artifacts versus inputs).\n- The runner validates source reads and absence of step5.done/decommission-list.txt, so the successful halt is supported by trace and filesystem evidence rather than prose alone (outcome.success; sniff.latest source-reads-grounded/no-phantom-completion; saved workspace inventory).\n\n## Negatives\n\n- The small procedure required 12 read_file calls and three writes, including a later runlog rewrite, over 5m54s. This is coordination/inference cost, not a correctness defect (toolUse.byTool; outcome.durationMs: 354208).\n- This fixed pass establishes the seeded file-based stop condition; it does not test concurrency, changing state between read and action, or remote operations. Those broader operational guarantees are outside the scenario (outcome.reason; saved runbook and state fixtures).\n\n## Performance and GPU fit\n\nElapsed 5.90 minutes; 1722 generated tokens at 9 tok/s, prefill 410.13 tok/s. Peak VRAM 10636/12227 MiB; process-tree RSS 14687 MiB; whole-system RAM 31080/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications.\n\nNative 0.1.57 CUDA; one launch, 65,536-token context, one slot, f16 KV and reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. No explicit OOM, context overflow or unexpected native exit was found. Shared GPU memory/paging is unmeasured. Performance stays separate from capability and advisory quality.\n\n## Tactical experiment — not applied\n\nOnly in a later isolated A/B, reduce the selected reasoning budget from 4096 to 2048 in ../gilde/data/chat-models/un/underdog-saluki-27b-iq2-mix/manifest.json, including its profile override. Keep sequential verification and write rules intact. Compare procedure correctness and token/latency cost with an unrelated document task; restore the baseline if either regresses.\n\n## Strategic experiment — not applied\n\nFor real runbook support, extend packages/service/src/tasks/manager.ts with reusable verification receipts that bind observed file version/hash, expected condition and subsequent permitted operation. This successful stop is a useful positive baseline; test changed-state invalidation on a different workflow before claiming general operational safety.\n\n## Generalization and next experiment\n\nRepetition 1 also passes fixed 9.5 and correctly halts at the same stale backup; repetition 2 retains that result. Contrast gate-script in complex work, where a manual completion label bypassed successful gate execution: explicit receipts and stop invariants matter across both workflows.\n\nTest each proposed change separately against this task family and an unrelated task before treating it as an improvement. No source, prompt, tuning, gate, model or campaign state was changed, and no capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/facts.json): pass, fixed 9.5, 11.89 min, 2.8 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/facts.json): pass, fixed 9.5, 1.25 min, 115.02 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-05-01-647Z-dcje/facts.json): pass, fixed 9.5, 1.43 min, 122.09 tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are historical context; no causal native-version or tuning improvement is inferred.\n\n## Strategic note\n\nThe verified halt is a positive example of following an explicit precondition and preserving trace evidence.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, historical-comparison.json, recording/chat-events.jsonl and the saved source/output files.\n" }, { "cellId": "r2-saluki-cuda-core-plan-and-estimate", "baselineCellId": "r2-saluki-cuda-core-plan-and-estimate", "suite": "core", "scenario": "plan-and-estimate", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T13:52:18.464Z", "finishedAt": "2026-11-05T13:55:50.485Z", "durationMs": 212021, "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 203848 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 3000, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06431304245484049 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 203848 }, { "source": "facts", "path": "outcome.durationMs", "value": 212021 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 35, "peakRssMb": 9330, "peakCpuPercent": 1197.1, "firstAtMs": 0, "lastAtMs": 200489 }, "gpu": { "available": true, "sampleCount": 35, "peakUtilPercent": 96, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 35, "peakUsedMb": 26365, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4520, "totalOutputTokens": 1578, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T13:55:46.849Z" } }, "source": "engine-log" }, "derived": { "durationMs": 201947, "meanTokensPerSec": 8.89, "genTokensPerSec": 8.89, "genTokens": 1578, "promptTokensPerSec": 565.58, "peakRssMb": 9330, "peakGpuUtilPercent": 96, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T13:52:24.034Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T13:52:23.934Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.05694580078125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.05694580078125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 212021, "timeoutMs": 3296703, "budgetUsedFraction": 0.06431304245484049 }, "timing": { "startedAt": "2026-11-05T13:52:18.464Z", "finishedAt": "2026-11-05T13:55:50.485Z", "timeToFirstArtifactMs": 203848, "timeToLastArtifactWriteMs": 203848, "timeToFirstTokenMs": 24534, "firstTurnTtftMs": 14322, "timeToFirstToolCallMs": 31470 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "read_files": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 207882, "filePath": "plan-and-estimate", "bytes": 3000, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } ], "latest": { "filePath": "plan-and-estimate", "bytes": 3000, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T13:52:18.468Z] [trial] id=plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet scenario=plan-and-estimate model=underdog-saluki-27b-iq2-mix", "[2026-11-05T13:52:18.468Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-FpNHjQ", "[2026-11-05T13:52:18.468Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "[2026-11-05T13:52:18.468Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T13:52:18.468Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T13:52:18.471Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T13:52:18.474Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T13:52:18.479Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-FpNHjQ", "[2026-11-05T13:52:18.479Z] [trial] throughput-scaled ceiling: 25m → 55m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T13:52:23.989Z] [trial] daemon spawned pid=8052 port=https://127.0.0.1:58816", "[2026-11-05T13:52:25.714Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T13:52:25.731Z] [trial] meester=jasper", "[2026-11-05T13:52:25.731Z] [trial] running scenario setup hook", "[2026-11-05T13:52:26.126Z] [trial] scenario setup complete", "[2026-11-05T13:52:26.127Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T13:52:26.193Z] [poll] initial digests hard=37009c0733b7ef9d soft=b136c92e14636a2a", "[2026-11-05T13:52:26.265Z] [poll] hard-progress: 37009c0733b7ef9d → 207404c71f74b3db after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T13:52:55.207Z] [poll] hard-progress: 207404c71f74b3db → 4dff0997a50b719e after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=32,stream=87) sniff=0/0B)", "[2026-11-05T13:55:46.346Z] [poll] terminal: success=true reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "[2026-11-05T13:55:46.881Z] [capture] dumped 1 session(s)", "[2026-11-05T13:55:46.886Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-05T13:55:46.959Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T13:55:50.483Z] [trial] daemon shut down", "[2026-11-05T13:55:50.492Z] [trial] PASS duration=212021ms reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)" ] }, "metrics": { "process": { "sampleCount": 35, "peakRssMb": 9330, "peakCpuPercent": 1197.1, "firstAtMs": 0, "lastAtMs": 200489 }, "gpu": { "available": true, "sampleCount": 35, "peakUtilPercent": 96, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 35, "peakUsedMb": 26365, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4520, "totalOutputTokens": 1578, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T13:55:46.849Z" } }, "source": "engine-log" }, "derived": { "durationMs": 201947, "meanTokensPerSec": 8.89, "genTokensPerSec": 8.89, "genTokens": 1578, "promptTokensPerSec": 565.58, "peakRssMb": 9330, "peakGpuUtilPercent": 96, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T13:52:18.464Z", "finishedAt": "2026-11-05T13:55:50.485Z", "timeToFirstArtifactMs": 203848, "timeToLastArtifactWriteMs": 203848, "timeToFirstTokenMs": 24534, "firstTurnTtftMs": 14322, "timeToFirstToolCallMs": 31470 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T13:52:24.034Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T13:52:23.934Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.05694580078125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.05694580078125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "scenarioId": "plan-and-estimate", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/workspace/harbourview-office-relocation/plan.md", "artifactBytes": 3006, "artifactSha256": "f25343bbb6419a74e0179f63807556b816e8e27efdfdcc81d1b144b80df894f9", "axisSource": "evals/src/scenarios/plan-and-estimate.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The concise eight-task plan covers all workstreams, internal owners, written floor-plan approval before mover booking and network readiness. Workshop completion is absent from final readiness dependencies, and construction depends only on the draft. Prose buffers do not repair those dependency gaps.", "reviewBasis": "Read the complete plan, brief and roster. Traced task dependencies and compared table commitments with assumptions/risks. No duration estimate was required by the supplied brief, so its absence is not penalized.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/workspace/harbourview-office-relocation/plan.md", "bytes": 3006, "sha256": "f25343bbb6419a74e0179f63807556b816e8e27efdfdcc81d1b144b80df894f9" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:00:59.918Z", "reviewId": "053", "cellId": "r2-saluki-cuda-core-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "scenarioId": "plan-and-estimate", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "baselineCellId": "r2-saluki-cuda-core-plan-and-estimate", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "3b9898311cba07f1600d3e11add2c1882a348b126afbdad0390bc12f93449f19", "evidenceSha256": "2a260f406bcc539f3d41724c79d024085e7c7e4b7f869a70808beedd783afefe", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/workspace/harbourview-office-relocation/plan.md", "bytes": 3006, "sha256": "f25343bbb6419a74e0179f63807556b816e8e27efdfdcc81d1b144b80df894f9", "shownBytes": 3006, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet\\workspace\\harbourview-office-relocation\\brief.md", "bytes": 732, "sha256": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "shownBytes": 732, "truncated": false } ], "scoreAxes": {}, "justification": "plan.md has the four required sections in order and a table with the exact columns. It has 8 rows, with owners from the roster (Ola not an owner) and valid earlier dependencies. T3 (movers) depends on T2 (sign-off). Floor plan, movers and network are all covered, and it is well under 1200 words.", "limitations": "I could not read team.md, so roster names are checked only against the prompt. Network-before-day-one is asserted by T8, not scheduled with dates. I did not check the reply format or the file-written-first order. The plan has no dates and does not show T6's dependency on T2. Source-only review.", "meanScore": null, "durationMs": 14523, "costUsd": 0.016302, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3262, "cache_read_input_tokens": 700, "output_tokens": 318, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3262, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 318, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3262, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3262 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 318, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3262, "webSearchRequests": 0, "costUSD": 0.016302, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3262, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - plan-and-estimate / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 3m 32s; budget used: 6.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 203848`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3000,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.06431304245484049`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 203848`; `facts.outcome.durationMs: 212021` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9330 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 96% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26365 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.89 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 4520 + 1578 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 203848`\n- `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3000,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.06431304245484049`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 203848`\n- `facts.outcome.durationMs: 212021`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 203848`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3000,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.06431304245484049`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 203848`; `facts.outcome.durationMs: 212021`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is preserved. This scenario has no declared axes; the advisory review is prose only.\n\nNo declared axes. The concise eight-task plan covers all workstreams, internal owners, written floor-plan approval before mover booking and network readiness. Workshop completion is absent from final readiness dependencies, and construction depends only on the draft. Prose buffers do not repair those dependency gaps.\n\nRead the complete plan, brief and roster. Traced task dependencies and compared table commitments with assumptions/risks. No duration estimate was required by the supplied brief, so its absence is not penalized.\n\n## Positives\n\n- The four ordered sections, three workstreams and word ceiling pass. The table assigns only roster members; the contractor advises/executes without owning a task (sniff.latest six signals; plan and team.md).\n- Movers booking depends on formal sign-off T2, and final readiness T8 depends on network T5 and physical move T7. The artifact was written in one call after a batch source read (toolUse.totalToolCalls: 2; read_files: 1; write_file: 1; plan dependency table).\n\n## Negatives\n\n- Workshop task T6 depends on draft T1 rather than sign-off T2; neither moving T7 nor readiness T8 depends on T6. The workshop can remain unfinished while the graph declares readiness. The overrun risk only mentions a buffer, not an enforced dependency (artifacts.otherFileCount; plan T6–T8 and risks).\n- The risk response permits rescheduling day one despite the brief’s September 30 deadline, without identifying this as a decision requiring stakeholder agreement. This is a contingency qualification, not a failure of the frozen structural gate (sniff.latest; plan risk response versus brief).\n\n## Performance and GPU fit\n\nElapsed 3.53 minutes; 1578 generated tokens at 8.89 tok/s, prefill 565.58 tok/s. Peak VRAM 10626/12227 MiB; process-tree RSS 9330 MiB; whole-system RAM 26365/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications.\n\nNative 0.1.57 CUDA; one launch, 65,536-token context, one slot, f16 KV and reasoning budget 4096. Dense FFN from 39/64 layers remains on GPU and 25 on CPU. No explicit OOM, context overflow or unexpected native exit was found. Shared GPU memory/paging is unmeasured. Performance stays separate from capability and advisory quality.\n\n## Tactical experiment — not applied\n\nIn a proposed generic packages/core/src/model-profile/behaviors/prompt-plan-consistency.ts block, ask the planner to trace every promised deliverable to the final readiness node and to flag any contingency that relaxes a deadline. Keep it independent of relocation/workstream names. A/B against an unrelated project plan and check dependency closure and token cost.\n\n## Strategic experiment — not applied\n\nAdd advisory dependency-graph analysis to packages/mcp/src/validate.ts for structured plans: detect orphaned required deliverables and mismatches between prose constraints and table edges. The earlier core and productivity plans show similar gaps, so investigate a generic validator rather than tuning this scenario’s expected strings.\n\n## Generalization and next experiment\n\nAll prior same-campaign plan entries scored fixed 10.0, yet dependency inconsistencies recur. Repetition 2 removes the earlier invented year and preserves the hard movers constraint, but still omits workshop readiness edges. Compare the runbook’s explicit stop rule, which the model followed correctly: structured enforceable prerequisites are more dependable than prose risk notes.\n\nTest each proposed change separately against this task family and an unrelated task before treating it as an improvement. No source, prompt, tuning, gate, model or campaign state was changed, and no capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/facts.json): pass, fixed 10, 8.99 min, 2.99 tok/s.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/facts.json): pass, fixed 10, 8.58 min, 2.88 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/plan-and-estimate/plan-and-estimate-gemma4-12b-q4-2026-10-28T08-20-45-068Z-a33h/facts.json): pass, fixed 10, 0.50 min, 111.68 tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are historical context; no causal native-version or tuning improvement is inferred.\n\n## Strategic note\n\nDependency closure remains weaker than the passing structural score across repeated plan outputs.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, historical-comparison.json, recording/chat-events.jsonl and the saved source/output files.\n" }, { "cellId": "r2-saluki-cuda-core-conflict-synthesis", "baselineCellId": "r2-saluki-cuda-core-conflict-synthesis", "suite": "core", "scenario": "conflict-synthesis", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T13:58:08.650Z", "finishedAt": "2026-11-05T14:16:58.902Z", "durationMs": 1130252, "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 270376 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 2498, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2857025857144157 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 30 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 270376 }, { "source": "facts", "path": "outcome.durationMs", "value": 1130252 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 189, "peakRssMb": 15400, "peakCpuPercent": 1205.9, "firstAtMs": 0, "lastAtMs": 1116886 }, "gpu": { "available": true, "sampleCount": 189, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 189, "peakUsedMb": 31199, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 221511, "totalOutputTokens": 4344, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 53620, "todayTokensCached": 19708, "todayTokensOut": 1663, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 53620, "totalTokensCached": 19708, "totalTokensOut": 1663, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13320, "outputTokens": 151, "cost": 0, "durationMs": 232976, "outputTokensPerSec": 8.343020043104492, "at": "2026-11-05T14:16:47.417Z" }, "lastUpdated": "2026-11-05T14:16:47.417Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T14:16:52.762Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1118238, "meanTokensPerSec": 8.55, "genTokensPerSec": 8.55, "genTokens": 4344, "promptTokensPerSec": 408.84, "peakRssMb": 15400, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 53620, "output": 1663 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T13:58:14.454Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T13:58:14.384Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 12, "maxContextFill": 0.171142578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.171142578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 1130252, "timeoutMs": 3956044, "budgetUsedFraction": 0.2857025857144157 }, "timing": { "startedAt": "2026-11-05T13:58:08.650Z", "finishedAt": "2026-11-05T14:16:58.902Z", "timeToFirstArtifactMs": 270376, "timeToLastArtifactWriteMs": 1118748, "timeToFirstTokenMs": 34400, "firstTurnTtftMs": 24830, "timeToFirstToolCallMs": 51564 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 30, "byTool": { "read_files": 1, "write_file": 2, "read_file": 24, "replace_in_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 274033, "filePath": "conflict-synthesis", "bytes": 2476, "score": 10, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 321541, "filePath": "conflict-synthesis", "bytes": 2476, "score": 11, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 517515, "filePath": "conflict-synthesis", "bytes": 2514, "score": 11, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 695709, "filePath": "conflict-synthesis", "bytes": 2507, "score": 11, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 885669, "filePath": "conflict-synthesis", "bytes": 2533, "score": 12, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry" }, { "atMs": 1123571, "filePath": "conflict-synthesis", "bytes": 2498, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } ], "latest": { "filePath": "conflict-synthesis", "bytes": 2498, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T13:58:08.653Z] [trial] id=conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf scenario=conflict-synthesis model=underdog-saluki-27b-iq2-mix", "[2026-11-05T13:58:08.654Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-Vt5bP7", "[2026-11-05T13:58:08.654Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T13:58:08.654Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "[2026-11-05T13:58:08.654Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T13:58:08.656Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T13:58:08.659Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T13:58:08.664Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-Vt5bP7", "[2026-11-05T13:58:08.664Z] [trial] throughput-scaled ceiling: 30m → 66m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T13:58:14.484Z] [trial] daemon spawned pid=30192 port=https://127.0.0.1:55065", "[2026-11-05T13:58:15.359Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T13:58:15.376Z] [trial] meester=jasper", "[2026-11-05T13:58:15.376Z] [trial] running scenario setup hook", "[2026-11-05T13:58:15.790Z] [trial] scenario setup complete", "[2026-11-05T13:58:15.791Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T13:58:15.867Z] [poll] initial digests hard=04ebe4e5928b0c1e soft=e35f996c74f7af10", "[2026-11-05T13:58:15.937Z] [poll] hard-progress: 04ebe4e5928b0c1e → bd67f99067743081 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T13:59:02.847Z] [poll] hard-progress: bd67f99067743081 → 2f4e84fe6d0769d5 after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=59,stream=190) sniff=0/0B)", "[2026-11-05T14:02:42.859Z] [poll] hard-progress: 2f4e84fe6d0769d5 → f32d4dfc367d6ef1 after 220s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=676,stream=2711) sniff=10/2476B)", "[2026-11-05T14:02:48.510Z] [poll] hard-progress: f32d4dfc367d6ef1 → 98a9a65f01eb5fba after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=682,stream=2726) sniff=10/2476B)", "[2026-11-05T14:03:30.232Z] [poll] hard-progress: 98a9a65f01eb5fba → 0b07e32b5ceb1925 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=791,stream=3048) sniff=11/2476B)", "[2026-11-05T14:03:48.063Z] [poll] hard-progress: 0b07e32b5ceb1925 → 70ef820584b8fd83 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=4,slot=804,stream=3064) sniff=11/2476B)", "[2026-11-05T14:04:05.797Z] [poll] hard-progress: 70ef820584b8fd83 → 281f8ae4bde69cc2 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=5,slot=816,stream=3082) sniff=11/2476B)", "[2026-11-05T14:04:23.561Z] [poll] hard-progress: 281f8ae4bde69cc2 → a8ff4e7b6c2c8587 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=6,slot=827,stream=3096) sniff=11/2476B)", "[2026-11-05T14:04:41.273Z] [poll] poisoned-session recovery: tamsin/857cf558 last turn aborted; sending one direct repair turn" ] }, "metrics": { "process": { "sampleCount": 189, "peakRssMb": 15400, "peakCpuPercent": 1205.9, "firstAtMs": 0, "lastAtMs": 1116886 }, "gpu": { "available": true, "sampleCount": 189, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 189, "peakUsedMb": 31199, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 221511, "totalOutputTokens": 4344, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 53620, "todayTokensCached": 19708, "todayTokensOut": 1663, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 53620, "totalTokensCached": 19708, "totalTokensOut": 1663, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13320, "outputTokens": 151, "cost": 0, "durationMs": 232976, "outputTokensPerSec": 8.343020043104492, "at": "2026-11-05T14:16:47.417Z" }, "lastUpdated": "2026-11-05T14:16:47.417Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T14:16:52.762Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1118238, "meanTokensPerSec": 8.55, "genTokensPerSec": 8.55, "genTokens": 4344, "promptTokensPerSec": 408.84, "peakRssMb": 15400, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 53620, "output": 1663 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T13:58:08.650Z", "finishedAt": "2026-11-05T14:16:58.902Z", "timeToFirstArtifactMs": 270376, "timeToLastArtifactWriteMs": 1118748, "timeToFirstTokenMs": 34400, "firstTurnTtftMs": 24830, "timeToFirstToolCallMs": 51564 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T13:58:14.454Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T13:58:14.384Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 12, "maxContextFill": 0.171142578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.171142578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-core-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "scenarioId": "conflict-synthesis", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/workspace/skylark-launch-synthesis/synthesis.md", "artifactBytes": 2504, "artifactSha256": "2046324a06760038561c6907c6a13b759ba2d0bc57749f5baa756b2b7f0d8df3", "axisSource": "evals/src/scenarios/conflict-synthesis.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The final synthesis correctly selects September 1, EUR 210,000 and Marcus, explains source precedence, and confines obsolete specific values to Conflicts. Open questions distinguish missing evidence from decisions. Earlier stale-value leakage was repaired after automatic engine recovery.", "reviewBasis": "Read the complete final synthesis and all six source/style files; compared the previous reviewed snapshot, edit trace and 13/13 final sniff. The separately captured native endpoint/exit recovery remains part of the run evidence." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:00.664Z", "reviewId": "054", "cellId": "r2-saluki-cuda-core-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "scenarioId": "conflict-synthesis", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "baselineCellId": "r2-saluki-cuda-core-conflict-synthesis", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "ed6fbf42b9a4564ccb85fde88671cb4abc0b59153cdf4af09f4856dc9c109b12", "evidenceSha256": "284ef9d7ce3d320469a65efa0f2fce79d02274856244988d9dd5ba2d986db871", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/workspace/skylark-launch-synthesis/synthesis.md", "bytes": 2504, "sha256": "2046324a06760038561c6907c6a13b759ba2d0bc57749f5baa756b2b7f0d8df3", "shownBytes": 2504, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The synthesis has the four required sections in order. It records three conflicts (date, budget, DRI), each quoting both values and file names with a stated winner. It cites files well over six times. Outside the conflicts section it uses only winning values, though the Overview names the superseded August date and 'old plan' without giving the old values.", "limitations": "Source files were not shown, so conflicts the brief missed can't be ruled out, and the Marcus, Iris, Deniz and 'onboarding revamp' details can't be checked. The Overview's mention of 'August date' is borderline. Source-only review with no execution.", "meanScore": null, "durationMs": 14747, "costUsd": 0.013882, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2652, "cache_read_input_tokens": 700, "output_tokens": 320, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2652, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 320, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2652, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2652 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 320, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2652, "webSearchRequests": 0, "costUSD": 0.013882, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2652, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - conflict-synthesis / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 18m 50s; budget used: 28.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 270376`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2498,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2857025857144157`; `facts.toolUse.totalToolCalls: 30`; `facts.timing.timeToFirstArtifactMs: 270376`; `facts.outcome.durationMs: 1130252` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15400 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31199 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.55 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 221511 + 4344 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 17% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 270376`\n- `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2498,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.2857025857144157`\n- `facts.toolUse.totalToolCalls: 30`\n- `facts.timing.timeToFirstArtifactMs: 270376`\n- `facts.outcome.durationMs: 1130252`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 270376`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2498,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2857025857144157`; `facts.toolUse.totalToolCalls: 30`; `facts.timing.timeToFirstArtifactMs: 270376`; `facts.outcome.durationMs: 1130252`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 8.5/10** is preserved. No declared axes; prose advisory review only.\n\nNo declared axes. The final synthesis correctly selects September 1, EUR 210,000 and Marcus, explains source precedence, and confines obsolete specific values to Conflicts. Open questions distinguish missing evidence from decisions. Earlier stale-value leakage was repaired after automatic engine recovery.\n\nRead the complete final synthesis and all six source/style files; compared the previous reviewed snapshot, edit trace and 13/13 final sniff. The separately captured native endpoint/exit recovery remains part of the run evidence.\n\n## Positives\n\n- All 13 signals pass: authoritative launch date, budget and ownership are selected with citations and explicit reconciliation; superseded specific values are confined to Conflicts (outcome.success; sniff.latest 13/13; final synthesis versus source files).\n- The repair removed old-budget leakage and the derived EUR30k amount from non-conflict prose. It preserved source-backed line items and open questions about cadence, budget history and migration start (toolUse.byTool.replace_in_file: 3; final artifact versus advisory-progress-2026-11-05T1400/synthesis.md).\n\n## Negatives\n\n- Delivery took 18m50s and 30 tool calls, including 24 individual source reads despite an initial batch read. Source rereading and bounded repair dominate the trace; no red flag was recorded (outcome.durationMs: 1130252; toolUse.byTool; toolUse.redFlags).\n- At 14:04:40 the native endpoint became unreachable; process 20316 exited at 14:04:41 with code3221226505, annotated expected/memory-pressure. The standard runner reloaded and continued. No explicit CUDA OOM or context overflow establishes the cause; this third such campaign incident remains a stability qualification even though the task ultimately passed (result.engineContext.launches:2; daemon.log; saved engine-recovery.json).\n- The gate rejected a correctly calculated EUR30k difference because it permits only amounts explicitly found in sources. That is distinct from the real stale-budget placement error. Preserve the frozen pass/fail trajectory without describing the arithmetic itself as wrong (earlier sniff; previous snapshot and source amounts).\n\n## Performance and GPU fit\n\nElapsed 18.84 minutes; 4344 recorded generated tokens at 8.55 tok/s, prefill 408.84 tok/s. Peak VRAM 10642/12227 MiB; process-tree RSS 15400 MiB; whole-system RAM 31199/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications and recorded token counts need not include an aborted unfinished turn.\n\nNative0.1.57 CUDA, 65,536-token context, one slot, f16KV, reasoning4096; dense FFN39/64 GPU layers and25 CPU layers. Shared GPU memory/paging unmeasured. Two launches: the standard runner automatically recovered the recorded endpoint failure/native exit and the model resumed. See advisory-progress-2026-11-05T1400/engine-recovery.json. No manual retry or environmental repair was performed. Maximum recorded context fill 17.1%, with no mid-turn compactions (continuity.sessions/compaction).\n\n## Tactical experiment — not applied\n\nIn proposed packages/core/src/model-profile/behaviors/prompt-evidence-conflicts.ts, add a generic revision instruction to separate selected current facts from historical alternatives and to check the full document after a local edit. Include a source-versus-derived-claim distinction. A/B on source reconciliation and an unrelated policy summary; measure stale leakage, legitimate derivations and reread cost.\n\n## Strategic experiment — not applied\n\nIn packages/inference/src/providers/llama-cpp/stdout-parser.ts and engine lifecycle telemetry, preserve the first endpoint failure, native exit code, supervisor eviction reason and memory samples as separate events. An expected eviction label must not erase a preceding endpoint failure. In evals/src/scenarios/conflict-synthesis.ts, a future separately versioned evaluator could distinguish attributed arithmetic from unsupported new budget assertions; do not relax or alter the current gate.\n\n## Generalization and next experiment\n\nThe prior core repetition also scored8.5 with31calls; the prior productivity copy scored10 with8calls. The final canonical facts are consistent, while coordination cost varies. Compare the incident report, whose citations still accompanied unsupported causal claims: explicit precedence rules help this synthesis task, but do not establish broad narrative grounding. The native recovery needs separate stability tracking from output competence.\n\nRun any proposed change separately against this task family and an unrelated task before recommending it. No source, prompt, tuning, gate or model change was made and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/facts.json): pass, fixed 10, 28.92min, 2.7tok/s.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/facts.json): pass, fixed 8.5, 55.01min, 2.84tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/conflict-synthesis/conflict-synthesis-gemma4-12b-q4-2026-10-28T08-40-57-167Z-gjl2/facts.json): pass, fixed 10, 0.60min, 122.83tok/s.\n\nSame-campaign repetitions share a frozen identity; older runs are contextual only. No causal native-version improvement is inferred.\n\n## Strategic note\n\nCorrect synthesis was delivered after automatic recovery. Keep endpoint/native stability separate from successful final content.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-edit-evidence.json, historical-comparison.json and the saved source/output files.\n" }, { "cellId": "r2-saluki-cuda-productivity-constrained-comms", "baselineCellId": "r2-saluki-cuda-productivity-constrained-comms", "suite": "productivity", "scenario": "constrained-comms", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T14:19:48.931Z", "finishedAt": "2026-11-05T14:23:02.709Z", "durationMs": 193778, "success": false, "reason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "constrained-comms", "score": 8, "bytes": 653, "failReason": "word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 67920 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 653, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words" } } ] }, "quality": { "score": 5, "ruleId": "Q_PARTIAL_UNKNOWN_MAX", "summary": "An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 67920 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 653, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07347414823063796 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 7 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 67920 }, { "source": "facts", "path": "outcome.durationMs", "value": 193778 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6, "band": "needs-tuning" }, "facts": { "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 32, "peakRssMb": 13233, "peakCpuPercent": 1193.9, "firstAtMs": 0, "lastAtMs": 182321 }, "gpu": { "available": true, "sampleCount": 32, "peakUtilPercent": 98, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 30067, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26318, "totalOutputTokens": 828, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 20398, "todayTokensCached": 11498, "todayTokensOut": 714, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 20398, "totalTokensCached": 11498, "totalTokensOut": 714, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5150, "outputTokens": 189, "cost": 0, "durationMs": 36912, "outputTokensPerSec": 8.257881697850346, "at": "2026-11-05T14:22:54.171Z" }, "lastUpdated": "2026-11-05T14:22:54.171Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T14:22:58.797Z" } }, "source": "engine-log" }, "derived": { "durationMs": 183646, "meanTokensPerSec": 8.33, "genTokensPerSec": 8.33, "genTokens": 828, "promptTokensPerSec": 516.79, "peakRssMb": 13233, "peakGpuUtilPercent": 98, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 20398, "output": 714 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T14:19:54.734Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T14:19:54.632Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wren": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 8, "maxContextFill": 0.093658447265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.093658447265625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words", "durationMs": 193778, "timeoutMs": 2637363, "budgetUsedFraction": 0.07347414823063796 }, "timing": { "startedAt": "2026-11-05T14:19:48.931Z", "finishedAt": "2026-11-05T14:23:02.709Z", "timeToFirstArtifactMs": 67920, "timeToLastArtifactWriteMs": 185240, "timeToFirstTokenMs": 36373, "firstTurnTtftMs": 26295, "timeToFirstToolCallMs": 42546 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 7, "byTool": { "read_files": 1, "write_file": 3, "read_file": 2, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 72005, "filePath": "constrained-comms", "bytes": 513, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 78 words, need ≥ 140 — the notice must be 140-220 words" }, { "atMs": 112976, "filePath": "constrained-comms", "bytes": 657, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words" }, { "atMs": 148201, "filePath": "constrained-comms", "bytes": 729, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 118 words, need ≥ 140 — the notice must be 140-220 words" }, { "atMs": 189322, "filePath": "constrained-comms", "bytes": 653, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words" } ], "latest": { "filePath": "constrained-comms", "bytes": 653, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T14:19:48.934Z] [trial] id=constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6 scenario=constrained-comms model=underdog-saluki-27b-iq2-mix", "[2026-11-05T14:19:48.934Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "[2026-11-05T14:19:48.934Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-constrained-comms-dZICaY", "[2026-11-05T14:19:48.934Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T14:19:48.934Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T14:19:48.937Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T14:19:48.942Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T14:19:48.947Z] [trial] throughput-scaled ceiling: 20m → 44m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T14:19:48.947Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-constrained-comms-dZICaY", "[2026-11-05T14:19:54.684Z] [trial] daemon spawned pid=42740 port=https://127.0.0.1:56916", "[2026-11-05T14:19:56.031Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T14:19:56.048Z] [trial] meester=florian", "[2026-11-05T14:19:56.048Z] [trial] running scenario setup hook", "[2026-11-05T14:19:56.450Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T14:19:56.449Z] [trial] scenario setup complete", "[2026-11-05T14:19:56.519Z] [poll] initial digests hard=c2410bc3698e6995 soft=82e730907de7205a", "[2026-11-05T14:19:56.600Z] [poll] hard-progress: c2410bc3698e6995 → 2d6d30242a3b35ba after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T14:20:31.610Z] [poll] hard-progress: 2d6d30242a3b35ba → 995f525fd7094e70 after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=26,stream=74) sniff=0/0B)", "[2026-11-05T14:21:01.072Z] [poll] hard-progress: 995f525fd7094e70 → 8afc1b9ec73ee8be after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=77,stream=332) sniff=8/513B)", "[2026-11-05T14:21:18.563Z] [poll] hard-progress: 8afc1b9ec73ee8be → d2b15154ef7a4379 after 17s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=3,slot=90,stream=351) sniff=8/513B)", "[2026-11-05T14:21:42.019Z] [poll] hard-progress: d2b15154ef7a4379 → 7e9a09b6b84af64c after 23s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=4,slot=148,stream=672) sniff=8/657B)", "[2026-11-05T14:21:53.748Z] [poll] hard-progress: 7e9a09b6b84af64c → 59104e06c4b96fd1 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=5,slot=160,stream=703) sniff=8/657B)", "[2026-11-05T14:22:17.258Z] [poll] hard-progress: 59104e06c4b96fd1 → 81bd660cd258c622 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=6,slot=208,stream=970) sniff=8/729B)", "[2026-11-05T14:22:58.281Z] [poll] terminal (scenario handoff): repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 106 words, need ≥ 140 — the notice must be 140-220 words", "[2026-11-05T14:22:58.825Z] [capture] dumped 1 session(s)" ] }, "metrics": { "process": { "sampleCount": 32, "peakRssMb": 13233, "peakCpuPercent": 1193.9, "firstAtMs": 0, "lastAtMs": 182321 }, "gpu": { "available": true, "sampleCount": 32, "peakUtilPercent": 98, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 32, "peakUsedMb": 30067, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26318, "totalOutputTokens": 828, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 20398, "todayTokensCached": 11498, "todayTokensOut": 714, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 20398, "totalTokensCached": 11498, "totalTokensOut": 714, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5150, "outputTokens": 189, "cost": 0, "durationMs": 36912, "outputTokensPerSec": 8.257881697850346, "at": "2026-11-05T14:22:54.171Z" }, "lastUpdated": "2026-11-05T14:22:54.171Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T14:22:58.797Z" } }, "source": "engine-log" }, "derived": { "durationMs": 183646, "meanTokensPerSec": 8.33, "genTokensPerSec": 8.33, "genTokens": 828, "promptTokensPerSec": 516.79, "peakRssMb": 13233, "peakGpuUtilPercent": 98, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 20398, "output": 714 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T14:19:48.931Z", "finishedAt": "2026-11-05T14:23:02.709Z", "timeToFirstArtifactMs": 67920, "timeToLastArtifactWriteMs": 185240, "timeToFirstTokenMs": 36373, "firstTurnTtftMs": 26295, "timeToFirstToolCallMs": 42546 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T14:19:54.734Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T14:19:54.632Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wren": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 8, "maxContextFill": 0.093658447265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.093658447265625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-constrained-comms", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "scenarioId": "constrained-comms", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/workspace/driftwater-outage-notice/customer-notice.md", "artifactBytes": 655, "artifactSha256": "3e12989b54687213427048080c2158da75b6e1510fa6350ac6484a1fce22a613", "axisSource": "evals/src/scenarios/constrained-comms.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The notice is calm and covers the true outage window, affected customers, payment-data disclosure, certificate fix, automatic credit and support contact. It is only106 words against140–220. Four completed repair attempts did not satisfy the length constraint, so the standard runner ended the trial.", "reviewBasis": "Read the final notice, incident brief, disclosure requirements, voice guide and stale draft. Inspected three full writes and one replacement, their feedback trajectory and the final word-band verdict. No numeric axes were invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:07.924Z", "reviewId": "055", "cellId": "r2-saluki-cuda-productivity-constrained-comms", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "scenarioId": "constrained-comms", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "baselineCellId": "r2-saluki-cuda-productivity-constrained-comms", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "ddcc5275f7caf65beb2166854da2fbdae6e7972f9ce5f7a396e9edbbe3a17e3e", "evidenceSha256": "97d100fa6c9cd8f2e209e88acf4a20fe0dd1e3605a6f57524cb36c4d877e63f6", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/workspace/driftwater-outage-notice/customer-notice.md", "bytes": 655, "sha256": "3e12989b54687213427048080c2158da75b6e1510fa6350ac6484a1fce22a613", "shownBytes": 655, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6\\workspace\\driftwater-outage-notice\\facts\\incident-brief.md", "bytes": 524, "sha256": "a65cd16d09c63d27126363a03f1bc664d26e4c6fc58eae192477d98ac3d02251", "shownBytes": 524, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6\\workspace\\driftwater-outage-notice\\facts\\legal-requirements.md", "bytes": 446, "sha256": "f15d48f9a7a37a2e3d855566ffd9cbdb9f0633a9f8c7f0756816b40a15815920", "shownBytes": 446, "truncated": false } ], "scoreAxes": {}, "justification": "Notice appears to meet the stated constraints: it has the June 30 date, 38 minutes, the exact sentence, the support email and the one-month credit, with no banned phrases. Facts match the brief. By my rough count it is about 105 words, below the 140–220 minimum, so it likely fails the word-count constraint.", "limitations": "The word count is my own manual estimate, not a tool-measured figure. The review is source-only, with no execution or transcript. It does not show that the model read the files first or that write_file was used. No numeric axes were declared.", "meanScore": null, "durationMs": 14291, "costUsd": 0.012736, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2428, "cache_read_input_tokens": 700, "output_tokens": 295, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2428, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 295, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2428, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2428 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 295, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2428, "webSearchRequests": 0, "costUSD": 0.012736, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2428, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - constrained-comms / underdog-saluki-27b-iq2-mix\n\n**Composite: 6.0 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 3m 14s; budget used: 7.3%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`; `facts.timing.timeToFirstArtifactMs: 67920`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":653,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...` |\n| Output quality (25%) | 5 / 10 | An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 67920`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":653,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.07347414823063796`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 67920`; `facts.outcome.durationMs: 193778` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **6.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13233 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 98% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30067 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.33 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 26318 + 828 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 9% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`\n- `facts.timing.timeToFirstArtifactMs: 67920`\n- `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":653,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n\n### Output quality - Q_PARTIAL_UNKNOWN_MAX\n\nAn authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 67920`\n- `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":653,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.07347414823063796`\n- `facts.toolUse.totalToolCalls: 7`\n- `facts.timing.timeToFirstArtifactMs: 67920`\n- `facts.outcome.durationMs: 193778`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`; `facts.timing.timeToFirstArtifactMs: 67920`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":653,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n- Quality: An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 67920`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":653,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.07347414823063796`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 67920`; `facts.outcome.durationMs: 193778`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **fail, 6/10** is preserved. No declared axes; prose advisory review only.\n\nNo declared axes. The notice is calm and covers the true outage window, affected customers, payment-data disclosure, certificate fix, automatic credit and support contact. It is only106 words against140–220. Four completed repair attempts did not satisfy the length constraint, so the standard runner ended the trial.\n\nRead the final notice, incident brief, disclosure requirements, voice guide and stale draft. Inspected three full writes and one replacement, their feedback trajectory and the final word-band verdict. No numeric axes were invented.\n\n## Positives\n\n- The notice uses the correct38-minute incident and12% impact, the required payment-data sentence and support address, and automatic one-month credit. It avoids the stale83-minute draft and banned guarantee phrases (sniff.latest eight passing signals; final artifact versus source pack).\n- The final wording replaces the initial stronger \"so this does not recur\" with \"less likely to recur\". It removes the intermediate speculative customer retry/abandonment sentence and keeps an accountable tone (recorded first write, replacement and final write; artifacts.otherFileCount).\n\n## Negatives\n\n- The final106-word notice fails the explicit140–220 band. Three full writes and one patch do not close the constraint, and bounded feedback exhausts four delivered/completed attempts (outcome.reason; sniff.latest.failReason; toolUse.byTool.write_file:3, replace_in_file:1).\n- The intermediate expansion speculated that customers retried or abandoned orders, then the final rewrite removed it and returned to an undersized notice. The revisions lacked a deterministic count check; no validation/execution call appears (toolUse.byTool; recording/chat-events.jsonl).\n\n## Performance and GPU fit\n\nElapsed 3.23 minutes; 828 recorded generated tokens at 8.33 tok/s, prefill 516.79 tok/s. Peak VRAM 10626/12227 MiB; process-tree RSS 13233 MiB; whole-system RAM 30067/31865 MiB (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications and recorded token counts need not include an aborted unfinished turn.\n\nNative0.1.57 CUDA, 65,536-token context, one slot, f16KV, reasoning4096; dense FFN39/64 GPU layers and25 CPU layers. Shared GPU memory/paging unmeasured. One launch, with no explicit CUDA OOM, context overflow or unexpected native-exit evidence found. The failure is unmet writing constraints, not inability to load or run. Maximum recorded context fill 9.4%, with no mid-turn compactions (continuity.sessions/compaction).\n\n## Tactical experiment — not applied\n\nIn a proposed generic packages/core/src/model-profile/behaviors/prompt-bounded-writing.ts, instruct bounded-writing tasks to target the midpoint of the supplied range, preserve supported facts, then count with an available validator before closure. Do not encode this scenario’s140–220 band or exact wording. Compare this task with an unrelated bounded brief and track compliance plus factual additions.\n\n## Strategic experiment — not applied\n\nExpose optional word-count/range checks through packages/mcp/src/validate.ts, returning the observed count and required delta without confusing semantic grounding with length. In evals/src/sniff-feedback.ts, retain the specific unmet constraint in repair receipts so edits can be checked for progress. Keep all current scenario gates and budgets fixed.\n\n## Generalization and next experiment\n\nRepetition1 also failed fixed6.0 at112words; repetition2 fails6.0 at106words. The repeat failure supports a general measurable-constraint repair issue. In contrast, the relocation plan respects its ceiling, while data-wrangle retains a sorting error after edits. These tasks favor deterministic contract checks over repeated unchecked rewrites.\n\nRun any proposed change separately against this task family and an unrelated task before recommending it. No source, prompt, tuning, gate or model change was made and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/facts.json): fail, fixed 6, 8.00min, 2.99tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/facts.json): pass, fixed 10, 0.69min, 136.68tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-11-50-881Z-lwv8/facts.json): pass, fixed 10, 0.48min, 127.21tok/s.\n\nSame-campaign repetitions share a frozen identity; older runs are contextual only. No causal native-version improvement is inferred.\n\n## Strategic note\n\nWord-band failure recurs across both repetitions; preserve the real useful notice while recording that the requested constraint was not met.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-edit-evidence.json, historical-comparison.json and the saved source/output files.\n" }, { "cellId": "r2-saluki-cuda-productivity-records-intake", "baselineCellId": "r2-saluki-cuda-productivity-records-intake", "suite": "productivity", "scenario": "records-intake", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T14:25:48.978Z", "finishedAt": "2026-11-05T16:14:11.758Z", "durationMs": 6502780, "success": true, "reason": "attendees.csv matches the declared schema with all golden rows (signals: csv-shape, record-schema, golden-rows)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1276387 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "records-intake", "bytes": 1134, "score": 3, "scoreMax": 3, "signals": [ "csv-shape", "record-schema", "golden-rows" ], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 22 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1276387 }, { "source": "facts", "path": "outcome.durationMs", "value": 6502780 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 1090, "peakRssMb": 15047, "peakCpuPercent": 2732839.5, "firstAtMs": 1, "lastAtMs": 6491205 }, "gpu": { "available": true, "sampleCount": 1090, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1090, "peakUsedMb": 30180, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26421, "totalOutputTokens": 52157, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 5662, "todayTokensCached": 0, "todayTokensOut": 538, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 5662, "totalTokensCached": 0, "totalTokensOut": 538, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5662, "outputTokens": 538, "cost": 0, "durationMs": 89745, "outputTokensPerSec": 9.011341100001824, "at": "2026-11-05T16:14:04.438Z" }, "lastUpdated": "2026-11-05T16:14:04.438Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T16:14:07.903Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6492616, "meanTokensPerSec": 8.2, "genTokensPerSec": 8.2, "genTokens": 52157, "promptTokensPerSec": 356.4, "peakRssMb": 15047, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 5662, "output": 538 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T14:25:54.644Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T14:25:54.544Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "griet": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 6, "maxContextFill": 0.08197021484375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.08197021484375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 2 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "attendees.csv matches the declared schema with all golden rows (signals: csv-shape, record-schema, golden-rows)", "durationMs": 6502780, "timeoutMs": 3296703, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-05T14:25:48.978Z", "finishedAt": "2026-11-05T16:14:11.758Z", "timeToFirstArtifactMs": 1276387, "timeToLastArtifactWriteMs": 6495460, "timeToFirstTokenMs": 24430, "firstTurnTtftMs": 14460, "timeToFirstToolCallMs": 45003 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 22, "byTool": { "list_dir": 3, "read_file": 11, "write_file": 2, "stat": 1, "replace_in_file": 5 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 6498419, "filePath": "records-intake", "bytes": 1134, "score": 3, "scoreMax": 3, "signals": [ "csv-shape", "record-schema", "golden-rows" ], "failReason": null } ], "latest": { "filePath": "records-intake", "bytes": 1134, "score": 3, "scoreMax": 3, "signals": [ "csv-shape", "record-schema", "golden-rows" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T14:25:48.982Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "[2026-11-05T14:25:48.982Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-records-intake-zmZe8K", "[2026-11-05T14:25:48.982Z] [trial] id=records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4 scenario=records-intake model=underdog-saluki-27b-iq2-mix", "[2026-11-05T14:25:48.982Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T14:25:48.982Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T14:25:48.985Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T14:25:48.989Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T14:25:48.993Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-records-intake-zmZe8K", "[2026-11-05T14:25:48.993Z] [trial] throughput-scaled ceiling: 25m → 55m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T14:25:54.569Z] [trial] daemon spawned pid=16668 port=https://127.0.0.1:63209", "[2026-11-05T14:25:56.114Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T14:25:56.130Z] [trial] running scenario setup hook", "[2026-11-05T14:25:56.130Z] [trial] meester=jasper", "[2026-11-05T14:25:56.550Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T14:25:56.548Z] [trial] scenario setup complete", "[2026-11-05T14:25:56.622Z] [poll] initial digests hard=5291ed769776c42b soft=1bfea272161beadf", "[2026-11-05T14:25:56.702Z] [poll] hard-progress: 5291ed769776c42b → bde23506c0d6c2f6 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T14:26:37.730Z] [poll] hard-progress: bde23506c0d6c2f6 → 4b8bb2174f0a7187 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=47,stream=122) sniff=0/0B)", "[2026-11-05T14:34:55.335Z] [poll] hard-progress: 4b8bb2174f0a7187 → dae1180f807e4f35 after 498s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=1412,stream=4153) sniff=0/0B)", "[2026-11-05T14:43:16.152Z] [poll] hard-progress: dae1180f807e4f35 → 2de39750eae6f831 after 501s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=2773,stream=8161) sniff=0/0B)", "[2026-11-05T14:47:08.540Z] [poll] hard-progress: 2de39750eae6f831 → 73be743c9e54344e after 232s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=3402,stream=11743) sniff=0/0B)", "[2026-11-05T14:47:20.328Z] [poll] hard-progress: 73be743c9e54344e → 5f3388772ea55bac after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=3419,stream=11797) sniff=0/0B)", "[2026-11-05T14:47:32.219Z] [poll] hard-progress: 5f3388772ea55bac → d7a3e5db3cb11059 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=3440,stream=11858) sniff=0/0B)", "[2026-11-05T14:47:49.810Z] [poll] hard-progress: d7a3e5db3cb11059 → c466b19869d89448 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=3478,stream=11979) sniff=0/0B)", "[2026-11-05T14:56:17.166Z] [poll] hard-progress: c466b19869d89448 → b8e097c10a330ae8 after 507s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=12,slot=4844,stream=16034) sniff=0/0B)" ] }, "metrics": { "process": { "sampleCount": 1090, "peakRssMb": 15047, "peakCpuPercent": 2732839.5, "firstAtMs": 1, "lastAtMs": 6491205 }, "gpu": { "available": true, "sampleCount": 1090, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1090, "peakUsedMb": 30180, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26421, "totalOutputTokens": 52157, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 5662, "todayTokensCached": 0, "todayTokensOut": 538, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 5662, "totalTokensCached": 0, "totalTokensOut": 538, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5662, "outputTokens": 538, "cost": 0, "durationMs": 89745, "outputTokensPerSec": 9.011341100001824, "at": "2026-11-05T16:14:04.438Z" }, "lastUpdated": "2026-11-05T16:14:04.438Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T16:14:07.903Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6492616, "meanTokensPerSec": 8.2, "genTokensPerSec": 8.2, "genTokens": 52157, "promptTokensPerSec": 356.4, "peakRssMb": 15047, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 5662, "output": 538 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T14:25:48.978Z", "finishedAt": "2026-11-05T16:14:11.758Z", "timeToFirstArtifactMs": 1276387, "timeToLastArtifactWriteMs": 6495460, "timeToFirstTokenMs": 24430, "firstTurnTtftMs": 14460, "timeToFirstToolCallMs": 45003 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T14:25:54.644Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T14:25:54.544Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "griet": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 6, "maxContextFill": 0.08197021484375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.08197021484375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 2 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-records-intake", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "scenarioId": "records-intake", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/workspace/conference-registration-intake/records/attendees.csv", "artifactBytes": 1134, "artifactSha256": "e86246bb7840cef1091999c3e5e6f7da1e66d6e922e8ebd218c65d42d66e01af", "axisSource": "evals/src/scenarios/records-intake.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. All 15 unique records match the source pack, including newest-registration dedupe, VIP upgrade, mixed dates and the quoted comma-bearing name. Delivery took 108 minutes after an unused faulty helper, repeated reads and automatic native-endpoint recovery. Correct CSV delivery does not demonstrate a working reusable transform.", "reviewBasis": "Read the full 1,134-byte CSV, all source/schema files, the final helper, and the edit/native recovery trace. Independently reconciled all 15 records and every field; no numeric axes invented." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:14.420Z", "reviewId": "056", "cellId": "r2-saluki-cuda-productivity-records-intake", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "scenarioId": "records-intake", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "baselineCellId": "r2-saluki-cuda-productivity-records-intake", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "53a3ccd8bbe2d693712cd87c57ad9ba0ee36f4b6f2e178051386eb9f423efe81", "evidenceSha256": "4bc785569616e13b697c0767f71f23944511a6878267844c84e23ba28b7c358c", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/workspace/conference-registration-intake/records/attendees.csv", "bytes": 1134, "sha256": "e86246bb7840cef1091999c3e5e6f7da1e66d6e922e8ebd218c65d42d66e01af", "shownBytes": 1134, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes are declared. The CSV has the exact header, 15 rows (above the 13 minimum), ISO dates, lowercase ticket types, and a quoted comma name (\"Kettle, Rosa\"). Quoting of \"Otis Bramble\" is harmless. Without the sources I can't verify dedup or the newest-wins rule.", "limitations": "Source files, schema.md and any script or transcript were not shown. Correctness of dedup, date parsing and ticket mapping is unverified. The \"runner-terminal\" disposition leaves the run's completion state unclear. Source-only review, no execution.", "meanScore": null, "durationMs": 14450, "costUsd": 0.012002, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2187, "cache_read_input_tokens": 700, "output_tokens": 318, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2187, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 318, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2187, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2187 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 318, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2187, "webSearchRequests": 0, "costUSD": 0.012002, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2187, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - records-intake / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 108m 23s; budget used: 100.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1276387`; `facts.sniff.latest: {\"filePath\":\"records-intake\",\"bytes\":1134,\"score\":3,\"scoreMax\":3,\"signals\":[\"csv-shape\",\"record-schema\",\"golden-rows\"],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 22`; `facts.timing.timeToFirstArtifactMs: 1276387`; `facts.outcome.durationMs: 6502780` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15047 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30180 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.2 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 26421 + 52157 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 8% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 2 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1276387`\n- `facts.sniff.latest: {\"filePath\":\"records-intake\",\"bytes\":1134,\"score\":3,\"scoreMax\":3,\"signals\":[\"csv-shape\",\"record-schema\",\"golden-rows\"],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 22`\n- `facts.timing.timeToFirstArtifactMs: 1276387`\n- `facts.outcome.durationMs: 6502780`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1276387`; `facts.sniff.latest: {\"filePath\":\"records-intake\",\"bytes\":1134,\"score\":3,\"scoreMax\":3,\"signals\":[\"csv-shape\",\"record-schema\",\"golden-rows\"],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 22`; `facts.timing.timeToFirstArtifactMs: 1276387`; `facts.outcome.durationMs: 6502780`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 8.5/10** is preserved. This scenario declares no qualitative axes; the review is prose only.\n\nNo declared axes. All 15 unique records match the source pack, including newest-registration dedupe, VIP upgrade, mixed dates and the quoted comma-bearing name. Delivery took 108 minutes after an unused faulty helper, repeated reads and automatic native-endpoint recovery. Correct CSV delivery does not demonstrate a working reusable transform.\n\n## Positives\n\n- The 1,134-byte CSV satisfies all three fixed signals and independently matches all 15 source records field for field. Mara's June 12 registration supersedes June 1; Otis is upgraded to VIP on June 13; both people in the two-person email are included. Rosa's comma-bearing name is correctly quoted (outcome.success; sniff.latest 3/3; advisory-record-audit.json).\n- IDs run A1–A15, emails are unique, all dates are valid ISO dates, and ticket types match the schema. No sorting beyond sequential IDs was requested; the final order is acceptable (sniff.latest.record-schema signal; final artifact and schema.md).\n- The standard runner's direct recovery turn delivered the CSV in one write, with no user answer required and no external retry (toolUse.byTool.write_file: 2, one helper and one CSV; autoAnswer.total: 0; recording/transcript.json).\n\n## Negatives\n\n- It took 108m23s, 52,157 recorded output tokens and 22 calls to deliver 15 rows. Eleven reads and five helper patches consumed work without creating the requested CSV. The repeated-read guard eventually aborted the turn (outcome.durationMs: 6502780; perf.derived.genTokens; toolUse.byTool; continuity.budget.toolRepeatAborts: 2).\n- The initial model-facing roster had 14 file/read tools and no execution tool. The prompt preferred an executed script; the model authored and repeatedly edited one without executing it. The helper is not a reusable successful transformation: ISO numeric dates are reversed, named-month parsing reads missing regex captures, matchAll uses a non-global regex, and the email regex still collects only the first address in a block (toolUse.byTool has no execution calls; daemon.log tool roster; saved helper). The final CSV was written directly and is correct independently of these helper defects.\n- At 16:12:17 UTC the identical-read guard aborted; at 16:12:32 the endpoint was unreachable; native PID 41300 then exited with code 3221226505, tagged expected/memory-pressure. The runner recovered and launched PID 47268. This is the fourth such endpoint/native sequence in the campaign, after doc-blocks, squisq and core conflict-synthesis. It must remain a stability qualification despite the successful task (result.engineContext.launches: 2; continuity.budget; advisory-engine-recovery.json).\n- The missing-output nudge identified seeded phone input as a wrong deliverable. That misleading near-miss classification should be tracked separately from the model's delay; the source file was not modified (sniff progression stayed at zero until delivery; recording/transcript.json; source hashes).\n\n## Performance and GPU fit\n\nNative 0.1.57 CUDA; 65,536-token context, one slot, f16 KV cache, reasoning budget 4,096. The offload plan kept dense FFNs for 39/64 layers on GPU and 25 on CPU. This was hybrid CPU/GPU execution, not full GPU residency.\n\nDecode averaged **8.2 tok/s**; prefill **356.4 tok/s**. Peak VRAM **10,644/12,227 MiB**, process-tree RSS **15,047 MiB**, whole-system RAM **30,180/31,865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications. Shared GPU memory and paging were not measured. The recorded CPU percentage is implausible and is not used as a utilization estimate.\n\nThe final captured session reports 8.2% context fill after rebuilding. Native slot logs reached **24757 tokens (37.8% of allocation)** earlier, so the final-session figure understates observed in-trial occupancy. No mid-turn compaction, explicit CUDA OOM or context overflow was recorded. The endpoint failure and exit annotation do not prove an OOM cause.\n\n## Tactical experiment — not applied\n\nIn evals/src/scenarios/records-intake.ts, make the execution preference in RECORDS_KICKOFF_MESSAGE/recordsRepairDirective conditional on the exposed execution roster, using the existing roster-aware pattern in packages/core/src/model-profile/behaviors/prompt-derive-by-execution.ts. When execution is absent, explicitly permit direct output followed by full schema/source verification. This is an eval-specific prompt experiment, not a claimed general fix; validate the same capability-aware wording in an unrelated data-import task, and then move it into shared instruction construction. Measure time to requested artifact, helper-only edits and field accuracy. Keep this campaign unchanged.\n\n## Strategic experiments — not applied\n\n- In packages/core/src/local-loop/tool-repeat-tracker.ts, evaluate a deliverable-aware progress signal that distinguishes helper churn from output progress. Issue one earlier actionable intervention when a requested output stays absent across repeated helper edits; do not reset progress solely because generation continues. A/B on this record import and unrelated report/game tasks, monitoring premature interruptions as well as completion cost.\n- In evals/src/scenarios/helpers.ts near-miss selection and evals/src/sniff-feedback.ts feedback construction, exclude seeded inputs from wrong-deliverable guesses unless they actually match the intended output contract. This run's phone-intake source is evidence of a generic false near-miss, not a reason to change the success gate.\n- In packages/inference/src/providers/llama-cpp/stdout-parser.ts and engine lifecycle telemetry, retain endpoint failure time separately from supervisor-requested eviction and native exit status. Correlate those events with memory samples before claiming a memory cause; expected=true alone does not make the preceding endpoint error harmless.\n\n## Generalization and next experiment\n\nThe first records-intake repetition failed at fixed 2.0 with no CSV; this repetition passes at 8.5 with correct full data, but only after excessive helper churn and recovery. The second data-wrangle repetition also spent over 100 minutes on a transform and still left a sorting error. Those diverse inputs support investigating execution-tool availability and observable output progress. In contrast, incident-report and plan trials delivered prose successfully without execution tools. Preserve that workflow when testing any generic remedy. No sampling, prompt, source, tool roster, budget or gate was changed here.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/facts.json): fail, fixed 2, 52.34 minutes, 2.8 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/facts.json): pass, fixed 10, 2.16 minutes, 143.1 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-15-17-692Z-396i/facts.json): pass, fixed 10, 2.17 minutes, 138.8 tok/s.\n\nSame-campaign repetitions share the frozen identity; older model/native/source runs are contextual only. No causal native-version benefit is inferred.\n\n## Strategic note\n\nThe final dataset is correct. A correct direct write after recovery does not establish a functioning script execution workflow or uninterrupted native stability. Keep those conclusions separate in the campaign assessment.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-record-audit.json, advisory-tool-evidence.json, advisory-engine-recovery.json, historical-comparison.json, daemon.log and the saved workspace sources/output. Earlier noArtifact checkpoints remain preserved as historical snapshots.\n" }, { "cellId": "r2-saluki-cuda-productivity-plan-and-estimate", "baselineCellId": "r2-saluki-cuda-productivity-plan-and-estimate", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T16:17:40.855Z", "finishedAt": "2026-11-05T16:20:51.733Z", "durationMs": 190878, "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 181446 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 3301, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.05789966521096987 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 181446 }, { "source": "facts", "path": "outcome.durationMs", "value": 190878 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 31, "peakRssMb": 9331, "peakCpuPercent": 1196.8, "firstAtMs": 1, "lastAtMs": 178852 }, "gpu": { "available": true, "sampleCount": 31, "peakUtilPercent": 99, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 31, "peakUsedMb": 27267, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4521, "totalOutputTokens": 1308, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T16:20:47.939Z" } }, "source": "engine-log" }, "derived": { "durationMs": 180436, "meanTokensPerSec": 8.53, "genTokensPerSec": 8.53, "genTokens": 1308, "promptTokensPerSec": 560.19, "peakRssMb": 9331, "peakGpuUtilPercent": 99, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T16:17:47.091Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T16:17:46.938Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0569305419921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0569305419921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 190878, "timeoutMs": 3296703, "budgetUsedFraction": 0.05789966521096987 }, "timing": { "startedAt": "2026-11-05T16:17:40.855Z", "finishedAt": "2026-11-05T16:20:51.733Z", "timeToFirstArtifactMs": 181446, "timeToLastArtifactWriteMs": 181446, "timeToFirstTokenMs": 26094, "firstTurnTtftMs": 15481, "timeToFirstToolCallMs": 34439 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "read_files": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 186576, "filePath": "plan-and-estimate", "bytes": 3301, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } ], "latest": { "filePath": "plan-and-estimate", "bytes": 3301, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T16:17:40.858Z] [trial] id=plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0 scenario=plan-and-estimate model=underdog-saluki-27b-iq2-mix", "[2026-11-05T16:17:40.858Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-I8WjcC", "[2026-11-05T16:17:40.858Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "[2026-11-05T16:17:40.858Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T16:17:40.858Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T16:17:40.861Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T16:17:40.865Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T16:17:40.870Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-I8WjcC", "[2026-11-05T16:17:40.870Z] [trial] throughput-scaled ceiling: 25m → 55m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T16:17:46.914Z] [trial] daemon spawned pid=54160 port=https://127.0.0.1:52644", "[2026-11-05T16:17:48.362Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T16:17:48.377Z] [trial] meester=rosalind", "[2026-11-05T16:17:48.377Z] [trial] running scenario setup hook", "[2026-11-05T16:17:48.780Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T16:17:48.779Z] [trial] scenario setup complete", "[2026-11-05T16:17:48.849Z] [poll] initial digests hard=fdeff0f69035a5ae soft=f78c58dc6bb7956d", "[2026-11-05T16:17:48.986Z] [poll] hard-progress: fdeff0f69035a5ae → b2e02eecdc3f9052 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T16:18:19.279Z] [poll] hard-progress: b2e02eecdc3f9052 → 97e5986664b5e755 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=32,stream=86) sniff=0/0B)", "[2026-11-05T16:20:47.431Z] [poll] terminal: success=true reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "[2026-11-05T16:20:47.976Z] [capture] dumped 1 session(s)", "[2026-11-05T16:20:47.979Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-05T16:20:48.069Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T16:20:51.731Z] [trial] daemon shut down", "[2026-11-05T16:20:51.741Z] [trial] PASS duration=190878ms reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)" ] }, "metrics": { "process": { "sampleCount": 31, "peakRssMb": 9331, "peakCpuPercent": 1196.8, "firstAtMs": 1, "lastAtMs": 178852 }, "gpu": { "available": true, "sampleCount": 31, "peakUtilPercent": 99, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 31, "peakUsedMb": 27267, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4521, "totalOutputTokens": 1308, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T16:20:47.939Z" } }, "source": "engine-log" }, "derived": { "durationMs": 180436, "meanTokensPerSec": 8.53, "genTokensPerSec": 8.53, "genTokens": 1308, "promptTokensPerSec": 560.19, "peakRssMb": 9331, "peakGpuUtilPercent": 99, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T16:17:40.855Z", "finishedAt": "2026-11-05T16:20:51.733Z", "timeToFirstArtifactMs": 181446, "timeToLastArtifactWriteMs": 181446, "timeToFirstTokenMs": 26094, "firstTurnTtftMs": 15481, "timeToFirstToolCallMs": 34439 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T16:17:47.091Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T16:17:46.938Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0569305419921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0569305419921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "scenarioId": "plan-and-estimate", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/workspace/harbourview-office-relocation/plan.md", "artifactBytes": 3313, "artifactSha256": "c5a52db34aef596fae76c4365222515a7ee030374f94825ae8f5a468e98fc5c4", "axisSource": "evals/src/scenarios/plan-and-estimate.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The eight-task plan uses internal owners, books movers after written floor-plan approval, and gates readiness on workshop and network completion. It ends at pre-move readiness: no task performs the physical move or verifies arrival. This limits completeness despite the fixed structural pass.", "reviewBasis": "Read the full final plan, brief and team roster; traced T1–T8 prerequisites and closure against the relocation objective. Read the two-call trace and final metrics. No duration estimate was required by the actual brief; no invented numeric axes." }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:15.348Z", "reviewId": "057", "cellId": "r2-saluki-cuda-productivity-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "scenarioId": "plan-and-estimate", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "baselineCellId": "r2-saluki-cuda-productivity-plan-and-estimate", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b34b38647e112f34403464b54ed05db76dc6e3075525cca18d600ce0ed1daaf7", "evidenceSha256": "f7494cc8f6fada6bd56cd26163fb89980d948030cfd5903f00da5f9c8837cccc", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/workspace/harbourview-office-relocation/plan.md", "bytes": 3313, "sha256": "c5a52db34aef596fae76c4365222515a7ee030374f94825ae8f5a468e98fc5c4", "shownBytes": 3313, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0\\workspace\\harbourview-office-relocation\\brief.md", "bytes": 732, "sha256": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "shownBytes": 732, "truncated": false } ], "scoreAxes": {}, "justification": "plan.md has the four sections in order and a table with the exact columns and 8 rows. Owners are all team members, never Ola. T4 (movers) depends on T3 (sign-off). Floor plan, movers and network are covered, and the document is well under 1200 words. Minor flaws: T5 and T8 don't show that the network is live before day one.", "limitations": "team.md was not shown, so owner names (Beatrix, Cas, Femke, Joris, Sanne) are checked only against the prompt's roster. Movers and move-day logistics are thin. The plan says \"all tasks above\" in Assumptions, before the table. Word count was estimated, not measured. No runtime verification.", "meanScore": null, "durationMs": 14633, "costUsd": 0.016974, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3375, "cache_read_input_tokens": 700, "output_tokens": 340, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3375, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 340, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3375, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3375 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 340, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3375, "webSearchRequests": 0, "costUSD": 0.016974, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3375, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - plan-and-estimate / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 3m 11s; budget used: 5.8%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 181446`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3301,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.05789966521096987`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 181446`; `facts.outcome.durationMs: 190878` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9331 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 99% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27267 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.53 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 4521 + 1308 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 181446`\n- `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3301,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.05789966521096987`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 181446`\n- `facts.outcome.durationMs: 190878`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 181446`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3301,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.05789966521096987`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 181446`; `facts.outcome.durationMs: 190878`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is preserved. No declared qualitative axes; prose review only.\n\nNo declared axes. The eight-task plan uses internal owners, books movers after written floor-plan approval, and gates readiness on workshop and network completion. It ends at pre-move readiness: no task performs the physical move or verifies arrival. This limits completeness despite the fixed structural pass.\n\n## Positives\n\n- All six signals pass: the four sections are ordered, the eight tasks use roster owners and earlier-row dependencies, all three workstreams appear, and the plan stays below the ceiling (sniff.latest: 6/6; final plan and team.md).\n- The key dependency is explicit: mover booking T4 follows written floor-plan sign-off T3. Workshop T5 and network configuration T7 must finish before readiness T8. Ola advises and executes under Sanne's ownership (sniff.latest.plan-structure signal; final plan table).\n- One batch read followed by one complete write delivered the plan in 3m11s with no red flags or user intervention (toolUse.totalToolCalls: 2; toolUse.byTool.read_files: 1, write_file: 1; outcome.durationMs: 190878; autoAnswer.total).\n\n## Negatives\n\n- The objective promises a relocated studio, but T8 is expressly a pre-move readiness check. T4 only books movers; no task packs, transports, receives or verifies the physical move. Readiness is useful but does not close the stated objective (outcome.success and sniff.latest show the structural gate's scope; final plan T4/T8 versus brief).\n- Budgets and exactly two meeting rooms are explicit assumptions, not established source facts. They should be confirmed before commitment. No durations were requested in the actual brief, so absent duration estimates are not treated as a contract failure (artifacts.otherFileCount; final assumptions versus brief.md).\n\n## Performance and GPU fit\n\n3.18 minutes; 1308 recorded generated tokens, **8.53 tok/s**, prefill 560.19 tok/s. Peak VRAM **10626/12227 MiB**, process-tree RSS **9331 MiB**, whole-system RAM **27267/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory).\n\nOne native 0.1.57 CUDA launch, 65,536-token allocation, one slot, f16 KV, reasoning budget 4,096. Dense FFNs for 39/64 layers on GPU and 25 on CPU. Whole-system RAM includes other applications; shared GPU memory and paging are unmeasured. No explicit OOM, context overflow or failed endpoint was found; the final SIGTERM is expected trial shutdown, not another recovery incident. Final captured session fill is 5.7%; this snapshot is not a measurement of peak KV use.\n\n## Tactical experiment — not applied\n\nIn a proposed packages/core/src/model-profile/behaviors/prompt-plan-consistency.ts, add one generic final check: map every promised objective to a task with an observable completion state, and distinguish preparation from performing the work. Do not encode relocation details or task IDs. A/B on a move plan and an unrelated launch plan, measuring objective coverage, task count and generation cost. This trial's positive fast delivery should remain intact.\n\n## Strategic experiment — not applied\n\nExtend advisory structured-plan checks in packages/mcp/src/validate.ts to expose objective-to-task coverage and whether the final node represents readiness or actual completion. Keep semantic judgments separate from the fixed structural gate. Repeated plans across this campaign pass the structure check while omitting different completion edges, which warrants a reusable coverage check rather than scenario-specific prompt memorization.\n\n## Generalization and next experiment\n\nThe second core copy scored 10 but left workshop completion outside the final dependency chain; this productivity copy correctly gates the workshop and network, yet omits the actual move. All four same-campaign plan trials pass structurally. Compare ops-runbook's successful halt after a failed prerequisite: stopping at readiness is correct only when the requested objective is readiness or a justified stop. Test the objective-closure check on both a full delivery task and an unrelated readiness-only task to detect overreach. No prompt, tuning, gate, source or model changes were made here.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/facts.json): pass, fixed 10, 3.53 minutes, 8.89 tok/s.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/facts.json): pass, fixed 10, 8.99 minutes, 2.99 tok/s.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/facts.json): pass, fixed 10, 8.58 minutes, 2.88 tok/s.\n\nThese comparison rows are evidence, not additional trials. Native-version improvements cannot be inferred from variable results with no controlled version comparison.\n\n## Strategic note\n\nThe structural pass is reliable across repetitions; semantic closure still varies. The fast generation and improved workshop dependency do not eliminate the missing physical-move task.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, historical-comparison.json, recording/chat-events.jsonl, daemon.log and workspace/harbourview-office-relocation/.\n" }, { "cellId": "r2-saluki-cuda-productivity-meeting-followup", "baselineCellId": "r2-saluki-cuda-productivity-meeting-followup", "suite": "productivity", "scenario": "meeting-followup", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T16:23:32.329Z", "finishedAt": "2026-11-05T16:28:33.551Z", "durationMs": 301222, "success": true, "reason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 110985 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "meeting-followup", "bytes": 2033, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07614222693175303 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 110985 }, { "source": "facts", "path": "outcome.durationMs", "value": 301222 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 50, "peakRssMb": 12233, "peakCpuPercent": 1199, "firstAtMs": 0, "lastAtMs": 288897 }, "gpu": { "available": true, "sampleCount": 50, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 50, "peakUsedMb": 29634, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 23592, "totalOutputTokens": 1924, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 6237, "todayTokensCached": 0, "todayTokensOut": 1084, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 6237, "totalTokensCached": 0, "totalTokensOut": 1084, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3473, "outputTokens": 542, "cost": 0, "durationMs": 107634, "outputTokensPerSec": 8.84501384171094, "at": "2026-11-05T16:27:10.957Z" }, "lastUpdated": "2026-11-05T16:27:10.957Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T16:28:29.300Z" } }, "source": "engine-log" }, "derived": { "durationMs": 290318, "meanTokensPerSec": 8.84, "genTokensPerSec": 8.84, "genTokens": 1924, "promptTokensPerSec": 459.73, "peakRssMb": 12233, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6237, "output": 1084 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T16:23:38.488Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T16:23:38.385Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "andrej": 1, "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 3, "maxContextFill": 0.094146728515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.094146728515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "durationMs": 301222, "timeoutMs": 3956044, "budgetUsedFraction": 0.07614222693175303 }, "timing": { "startedAt": "2026-11-05T16:23:32.329Z", "finishedAt": "2026-11-05T16:28:33.551Z", "timeToFirstArtifactMs": 110985, "timeToLastArtifactWriteMs": 295886, "timeToFirstTokenMs": 30349, "firstTurnTtftMs": 19957, "timeToFirstToolCallMs": 44059 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 8, "byTool": { "read_file": 4, "write_file": 3, "list_dir": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 296435, "filePath": "meeting-followup", "bytes": 2033, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } ], "latest": { "filePath": "meeting-followup", "bytes": 2033, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T16:23:32.332Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "[2026-11-05T16:23:32.332Z] [trial] id=meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu scenario=meeting-followup model=underdog-saluki-27b-iq2-mix", "[2026-11-05T16:23:32.332Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-meeting-followup-cU2jDK", "[2026-11-05T16:23:32.332Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T16:23:32.333Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T16:23:32.335Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T16:23:32.339Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T16:23:32.344Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-meeting-followup-cU2jDK", "[2026-11-05T16:23:32.345Z] [trial] throughput-scaled ceiling: 30m → 66m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T16:23:38.481Z] [trial] daemon spawned pid=47572 port=https://127.0.0.1:49181", "[2026-11-05T16:23:39.808Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T16:23:39.825Z] [trial] meester=tamsin", "[2026-11-05T16:23:39.825Z] [trial] running scenario setup hook", "[2026-11-05T16:23:40.276Z] [trial] scenario setup complete", "[2026-11-05T16:23:40.277Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T16:23:40.367Z] [poll] initial digests hard=c2fe929c211732ba soft=f151572a88903e72", "[2026-11-05T16:23:40.510Z] [poll] hard-progress: c2fe929c211732ba → fe97c699be08484f after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T16:24:21.504Z] [poll] hard-progress: fe97c699be08484f → 75765356be51268d after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=38,stream=85) sniff=0/0B)", "[2026-11-05T16:25:20.751Z] [poll] hard-progress: 75765356be51268d → 7dad98ee53578432 after 59s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=3,slot=206,stream=1062) sniff=0/0B)", "[2026-11-05T16:25:26.382Z] [poll] hard-progress: 7dad98ee53578432 → 2be5f7e5ab385585 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=214,stream=1091) sniff=1/1726B)", "[2026-11-05T16:25:55.921Z] [poll] hard-progress: 2be5f7e5ab385585 → d7af5c30955b8a60 after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=269,stream=1236) sniff=1/1726B)", "[2026-11-05T16:26:01.759Z] [poll] hard-progress: d7af5c30955b8a60 → 60dbcc956226a487 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=277,stream=1258) sniff=1/1726B)", "[2026-11-05T16:27:12.387Z] [poll] hard-progress: 60dbcc956226a487 → e0e8740fb4cddb9c after 71s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=456,stream=2264) sniff=1/1726B)", "[2026-11-05T16:28:28.764Z] [poll] terminal: success=true reason=meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "[2026-11-05T16:28:29.320Z] [capture] dumped 2 session(s)" ] }, "metrics": { "process": { "sampleCount": 50, "peakRssMb": 12233, "peakCpuPercent": 1199, "firstAtMs": 0, "lastAtMs": 288897 }, "gpu": { "available": true, "sampleCount": 50, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 50, "peakUsedMb": 29634, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 23592, "totalOutputTokens": 1924, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 6237, "todayTokensCached": 0, "todayTokensOut": 1084, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 6237, "totalTokensCached": 0, "totalTokensOut": 1084, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3473, "outputTokens": 542, "cost": 0, "durationMs": 107634, "outputTokensPerSec": 8.84501384171094, "at": "2026-11-05T16:27:10.957Z" }, "lastUpdated": "2026-11-05T16:27:10.957Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T16:28:29.300Z" } }, "source": "engine-log" }, "derived": { "durationMs": 290318, "meanTokensPerSec": 8.84, "genTokensPerSec": 8.84, "genTokens": 1924, "promptTokensPerSec": 459.73, "peakRssMb": 12233, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6237, "output": 1084 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T16:23:32.329Z", "finishedAt": "2026-11-05T16:28:33.551Z", "timeToFirstArtifactMs": 110985, "timeToLastArtifactWriteMs": 295886, "timeToFirstTokenMs": 30349, "firstTurnTtftMs": 19957, "timeToFirstToolCallMs": 44059 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T16:23:38.488Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T16:23:38.385Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "andrej": 1, "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 3, "maxContextFill": 0.094146728515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.094146728515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-meeting-followup", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "scenarioId": "meeting-followup", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/meeting-brief.md", "artifactKind": "markdown", "artifactBytes": 1732, "artifactSha256": "ec12314025683a01f310f329639b32633adb2693d239ac766fcdeb5321eead59", "supportingArtifactPaths": [ "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/action-items.csv" ], "axisSource": "evals/src/scenarios/meeting-followup.ts", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ], "scoreAxes": { "decisionClarity": 7, "grounding": 7, "actionability": 7, "brevity": 7 }, "meanScore": 7, "justification": "Clear current decisions, four accurate actions, the field-map dependency, and open annual-plan/rollback questions. The prose turns a scheduling deadline into a review due date and adds an unagreed scope prerequisite. The CSV remains correct. Stale proposals and the scheduling action are repeated across sections.", "reviewBasis": "Read the entire final Markdown and CSV, authoritative transcript, roster and stale agenda. Compared all four CSV rows and inspected the eight completed tool calls and native lifecycle log. Model identity was visible in paths.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/meeting-brief.md", "bytes": 1732, "sha256": "ec12314025683a01f310f329639b32633adb2693d239ac766fcdeb5321eead59" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/action-items.csv", "bytes": 307, "sha256": "2c081c4b560ff6b6fd8ef6f24f9d4f6031888754ec558fd15adea0802d6ca31c" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:24.463Z", "reviewId": "058", "cellId": "r2-saluki-cuda-productivity-meeting-followup", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "scenarioId": "meeting-followup", "status": "scored", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "baselineCellId": "r2-saluki-cuda-productivity-meeting-followup", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "fac01b9581b2288fe1987f760cb30f0584a73ac72a44cebcfe77062d6dbcb267", "evidenceSha256": "105b2bec131fb33c01ae57b41e77271919a4ebe058abde1cd9abc69ac17ee1d2", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/meeting-brief.md", "bytes": 1732, "sha256": "ec12314025683a01f310f329639b32633adb2693d239ac766fcdeb5321eead59", "shownBytes": 1732, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu\\workspace\\orchid-launch-follow-up\\meeting\\old-agenda.md", "bytes": 262, "sha256": "8a895418b408627cf5fd7367a45341bedc5fc29515c61dac05de3b47f6fdde38", "shownBytes": 262, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu\\workspace\\orchid-launch-follow-up\\meeting\\roster.md", "bytes": 223, "sha256": "8f77db20d7700521aecdf6cddcbbe807ebf99bae5c2043a558fd5f96831cad69", "shownBytes": 223, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu\\workspace\\orchid-launch-follow-up\\meeting\\transcript.md", "bytes": 977, "sha256": "7a4039bda8b643bfdfb3f8d1208a5c7835fd3b13f280867db9b6915992e0800a", "shownBytes": 977, "truncated": false } ], "scoreAxes": { "decisionClarity": 8, "grounding": 8, "actionability": 7, "brevity": 8 }, "justification": "The brief has the required H2 order. It separates the decisions (9/14, EU-only, one overnight batch) from the stale proposals. Annual plan and rollback owner stay open. Owners, dates and the A2→A1 dependency match the transcript and roster. Minor flaw: scheduling the annual-plan review is listed as a decision, though it is an action. The legal-review risk is captured.", "limitations": "action-items.csv was not among the shown artifacts, so its exact header, four rows, empty depends_on cells and status values could not be verified. Scores rest on the brief's action table alone. No runtime or rendering checks.", "meanScore": 7.75, "durationMs": 16491, "costUsd": 0.020656599999999997, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3552, "cache_read_input_tokens": 846, "output_tokens": 636, "output_tokens_details": { "thinking_tokens": 287 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3552, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 636, "cache_read_input_tokens": 846, "cache_creation_input_tokens": 3552, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3552 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 636, "cacheReadInputTokens": 846, "cacheCreationInputTokens": 3552, "webSearchRequests": 0, "costUSD": 0.020656599999999997, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 287, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3552, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - meeting-followup / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 5m 1s; budget used: 7.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 110985`; `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2033,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.07614222693175303`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 110985`; `facts.outcome.durationMs: 301222` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12233 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29634 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.84 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 23592 + 1924 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 9% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 110985`\n- `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2033,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.07614222693175303`\n- `facts.toolUse.totalToolCalls: 8`\n- `facts.timing.timeToFirstArtifactMs: 110985`\n- `facts.outcome.durationMs: 301222`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 110985`; `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2033,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.07614222693175303`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 110985`; `facts.outcome.durationMs: 301222`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline judge\n\nFixed result **pass, 10/10** remains unchanged. Advisory mean **7/10**, from codex-inline/current-session, measures a separate question.\n\n| Declared axis | Score | Evidence |\n|---|---:|---|\n| decisionClarity | 7 | Decisions, actions, open questions and risks have clear sections; the scheduling action is also listed as a decision. |\n| grounding | 7 | Dates, scope, migration mode and CSV owners match; prose upgrades the scheduling deadline and adds an unagreed dependency. |\n| actionability | 7 | Four usable CSV actions with A2 depending on A1; prose ambiguity about A4's deadline can mislead execution. |\n| brevity | 7 | Short and scannable, with repeated stale-agenda facts and the scheduling action across sections. |\n\nClear current decisions, four accurate actions, the field-map dependency, and open annual-plan/rollback questions. The prose turns a scheduling deadline into a review due date and adds an unagreed scope prerequisite. The CSV remains correct. Stale proposals and the scheduling action are repeated across sections.\n\n## Positives\n\n- All 13 reconciliation gates pass. September 14, EU-only scope and overnight migration are retained; annual-plan inclusion and rollback ownership remain open, and legal review remains unscheduled (outcome.success; sniff.latest signals; transcript and final brief).\n- All four CSV rows match their source action, owner and date. A2 correctly depends on A1; the other dependency cells are empty, and Casey is not assigned (sniff.latest action-row-a1 through action-row-a4; roster and action-items.csv).\n- Eight calls, no red flags and no user intervention deliver both outputs in 5m01s (toolUse.totalToolCalls; toolUse.redFlags; autoAnswer.total; outcome.durationMs).\n\n## Negatives\n\n- The Open questions section says the review is due August 30, although the transcript only commits Jordan to schedule it by that date. The last risk bullet also turns A4 into a decision that must be resolved before scope finalization. These stronger claims are absent from the source; the CSV itself correctly says Schedule annual-plan decision review (sniff.latest passes the narrower action checks; final brief versus transcript lines 17–18).\n- The brief is written twice with identical content before the missing CSV is written. A missing-deliverable nudge arrives after the second brief write. This is modest unnecessary work, not a broken final dataset (toolUse.byTool.write_file: 3; read_file: 4; advisory-artifact-review.json edit trace; log.txt).\n- The preamble and several sections repeat the superseded agenda, while A4 appears under Decisions and Action items. The final brief is readable but could be tighter (artifacts.otherFileCount; final Markdown).\n\n## Performance and GPU fit\n\nElapsed **5m01s**, 1,924 recorded generated tokens at **8.84 tok/s**, prefill **459.73 tok/s**. Peak VRAM **10,628/12,227 MiB**, process-tree RSS **12,233 MiB**, whole-system RAM **29,634/31,865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory).\n\nOne native 0.1.57 CUDA launch, 65,536-token allocation, one slot, f16 KV and 4,096 reasoning tokens. Dense FFNs for 39/64 layers on GPU, 25 on CPU. Final captured context fill is 9.4%, with no mid-turn compaction. No failed endpoint, explicit OOM or context overflow was found; the native SIGTERM is ordinary trial shutdown. Shared GPU memory/paging is unmeasured and whole-system RAM includes other applications.\n\n## Tactical experiment — not applied\n\nIn a proposed packages/core/src/model-profile/behaviors/prompt-commitment-fidelity.ts, add a short generic constraint to preserve the action verb and its deadline: scheduling a review is distinct from holding it or deciding its outcome. Compare a meeting follow-up and an unrelated incident-action report, measuring strengthened commitments and extra tokens. Do not embed this meeting's names, dates or expected wording.\n\n## Strategic experiment — not applied\n\nIn packages/mcp/src/validate.ts, explore advisory cross-artifact consistency for action registers and prose summaries: retain action, owner, dependency, deadline and whether the verb is schedule, perform or decide. The exact CSV versus stronger prose in this trial provides a concrete consistency case. In evals/src/sniff-feedback.ts, keep missing-output feedback targeted to the absent file so a valid sibling is not needlessly rewritten. Keep the frozen gates and campaign untouched.\n\n## Generalization and next experiment\n\nThe prior repetition also passed fixed 10 and scored advisory 7.5; this one retains useful reconciliation but adds deadline/dependency ambiguity, hence 7.0. The plan trials similarly pass structural checks while final objective closure varies. Contrast ops-runbook, whose explicit prerequisite was correctly enforced: proposal, readiness, execution and commitment status need distinct treatment across task types. Test the proposed commitment check on both source-faithful follow-ups and plans where new proposed actions are legitimately requested.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/facts.json): pass, fixed 10, 13.57 minutes, 2.98 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/facts.json): pass, fixed 10, 2.83 minutes, 130.36 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-24-12-830Z-kar9/facts.json): pass, fixed 10, 2.44 minutes, 139.08 tok/s.\n\nOlder model/native/source identities are contextual, not controlled comparisons. No native-version improvement is inferred.\n\n## Strategic note\n\nThe fixed pass and accurate register coexist with a small but consequential prose strengthening of a commitment. Preserve both conclusions rather than treating successful source reconciliation as exhaustive grounding.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, historical-comparison.json, recording/chat-events.jsonl, daemon.log and the source/output files under workspace/orchid-launch-follow-up/.\n" }, { "cellId": "r2-saluki-cuda-productivity-craftbook-spreadsheet-model", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-spreadsheet-model", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T16:31:03.221Z", "finishedAt": "2026-11-05T17:46:00.563Z", "durationMs": 4497342, "success": true, "reason": "craftbook-spreadsheet-model passed 6 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 325556 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-spreadsheet-model", "bytes": 3148, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 0, "ruleId": "E_BOTH_SEVERE", "summary": "The trial exhausted its budget while making at least thirty tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 39 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 325556 }, { "source": "facts", "path": "outcome.durationMs", "value": 4497342 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8, "band": "ship-ready" }, "facts": { "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 753, "peakRssMb": 14189, "peakCpuPercent": 1275.4, "firstAtMs": 0, "lastAtMs": 4484333 }, "gpu": { "available": true, "sampleCount": 753, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 753, "peakUsedMb": 30770, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 86138, "totalOutputTokens": 36981, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 86046, "todayTokensCached": 79713, "todayTokensOut": 9395, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 86046, "totalTokensCached": 79713, "totalTokensOut": 9395, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8213, "cachedInputTokens": 7747, "outputTokens": 214, "cost": 0, "durationMs": 141356, "outputTokensPerSec": 9.008208126993912, "at": "2026-11-05T17:45:53.151Z" }, "lastUpdated": "2026-11-05T17:45:53.151Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T17:45:55.245Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4485675, "meanTokensPerSec": 8.87, "genTokensPerSec": 8.87, "genTokens": 36981, "promptTokensPerSec": 378.94, "peakRssMb": 14189, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 86046, "output": 9395 } } } }, "craftbook": { "craftbookId": "spreadsheet-model", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 6, "activationEvents": 5, "completed": 4, "gateApprovals": 4, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T16:31:10.892Z", "completedAt": "2026-11-05T16:36:28.978Z", "ms": 318086, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "4c52fc3e-ce05-4ee0-a701-61dcde11a0b7" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "build", "activatedAt": "2026-11-05T17:07:50.476Z", "completedAt": "2026-11-05T17:27:02.891Z", "ms": 1152415, "gateRejections": 0, "toolCalls": 19, "sessionIds": [ "7368d88e-b4df-4e0e-a62c-18614dad9c96", "4e6de60d-3ed6-4a1a-90ad-2ab4821acf83" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T17:27:02.893Z", "completedAt": "2026-11-05T17:43:26.428Z", "ms": 983535, "gateRejections": 1, "toolCalls": 14, "sessionIds": [ "7df37340-bfc4-4cee-bdc2-35442932bc06", "facbc00d-6516-47ef-987f-4f3f86294342" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T17:43:26.430Z", "completedAt": "2026-11-05T17:45:53.107Z", "ms": 146677, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "df8d7677-5b24-4d0b-ba1d-b87a922bf011" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T16:31:09.036Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T16:31:08.937Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 650810.5 }, "sessions": { "total": 6, "taskScoped": 6, "byGezel": { "matej": 2, "mats": 1, "yerin": 3 }, "perTask": { "spreadsheet-financial-model-workflow-eval/1": 6 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 1, "maxMessages": 2, "maxContextFill": 0.1291961669921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1291961669921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 3, "citations": [ "knowledge://bendyline/handboek/model-scorecard", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/ops-runbook" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-spreadsheet-model passed 6 deterministic craftbook checks", "durationMs": 4497342, "timeoutMs": 3956044, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-05T16:31:03.221Z", "finishedAt": "2026-11-05T17:46:00.563Z", "timeToFirstArtifactMs": 325556, "timeToLastArtifactWriteMs": 4343114, "timeToFirstTokenMs": 33159, "firstTurnTtftMs": 19863, "timeToFirstToolCallMs": 60726 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Planner", "Developer", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 39, "byTool": { "read_file": 14, "write_artifact": 4, "read_artifact": 6, "write_file": 1, "write_task_note": 4, "validate": 3, "advance_task_step": 2, "list_artifacts": 1, "replace_lines": 1, "replace_in_file": 2, "read_task_notes": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/spreadsheet-financial-model-workflow-eval/index.html", "finalBytes": 13816, "growth": [] } ], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 7889, "filePath": "craftbook-spreadsheet-model", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, source/brief.md, source/records.csv" }, { "atMs": 66374, "filePath": "craftbook-spreadsheet-model", "bytes": 0, "score": 2, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook spreadsheet-model has not finished: its gates have not accepted the result yet" }, { "atMs": 326438, "filePath": "craftbook-spreadsheet-model", "bytes": 3148, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook spreadsheet-model has not finished: its gates have not accepted the result yet" }, { "atMs": 1137830, "filePath": "craftbook-spreadsheet-model", "bytes": 3148, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook spreadsheet-model has not finished: its gates have not accepted the result yet" }, { "atMs": 4491511, "filePath": "craftbook-spreadsheet-model", "bytes": 3148, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-spreadsheet-model", "bytes": 3148, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T16:31:03.224Z] [trial] id=craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1 scenario=craftbook-spreadsheet-model model=underdog-saluki-27b-iq2-mix", "[2026-11-05T16:31:03.224Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "[2026-11-05T16:31:03.224Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-spreadsheet-model-qfGwRq", "[2026-11-05T16:31:03.224Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T16:31:03.224Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T16:31:03.228Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T16:31:03.232Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T16:31:03.240Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-spreadsheet-model-qfGwRq", "[2026-11-05T16:31:03.240Z] [trial] throughput-scaled ceiling: 30m → 66m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T16:31:08.982Z] [trial] daemon spawned pid=33316 port=https://127.0.0.1:57115", "[2026-11-05T16:31:10.421Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T16:31:10.437Z] [trial] meester=nadia", "[2026-11-05T16:31:10.437Z] [trial] running scenario setup hook", "[2026-11-05T16:31:11.027Z] [trial] scenario setup complete", "[2026-11-05T16:31:11.028Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T16:31:11.072Z] [poll] initial digests hard=a1a51859d63c1259 soft=6b95cddf71bbfd9c", "[2026-11-05T16:31:11.148Z] [poll] hard-progress: a1a51859d63c1259 → 9da05d622eabcd0d after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T16:31:16.466Z] [poll] hard-progress: 9da05d622eabcd0d → c7cc63391ecaf0b0 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T16:32:09.658Z] [poll] hard-progress: c7cc63391ecaf0b0 → 1e768aeac3b70630 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=90,stream=244) sniff=2/0B)", "[2026-11-05T16:36:29.705Z] [poll] hard-progress: 1e768aeac3b70630 → 8da46a66bc7faae9 after 260s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=4,slot=859,stream=3365) sniff=3/3148B)", "[2026-11-05T16:37:17.034Z] [poll] hard-progress: 8da46a66bc7faae9 → 35da2bf23b541637 after 47s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=901,stream=3467) sniff=3/3148B)", "[2026-11-05T16:50:01.129Z] [poll] hard-progress: 35da2bf23b541637 → e28b3fdbe6c97f8b after 764s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=3092,stream=14162) sniff=5/3148B)", "[2026-11-05T16:50:31.049Z] [poll] hard-progress: e28b3fdbe6c97f8b → a22fa63bf2898de9 after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=10,slot=3166,stream=14513) sniff=5/3148B)", "[2026-11-05T16:50:37.084Z] [poll] hard-progress: a22fa63bf2898de9 → 2616d0a59ff3b26a after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=11,slot=3179,stream=14550) sniff=5/3148B)", "[2026-11-05T16:50:48.692Z] [poll] hard-progress: 2616d0a59ff3b26a → 7027d697f9d4706c after 12s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=12,slot=3197,stream=14610) sniff=5/3148B)" ] }, "metrics": { "process": { "sampleCount": 753, "peakRssMb": 14189, "peakCpuPercent": 1275.4, "firstAtMs": 0, "lastAtMs": 4484333 }, "gpu": { "available": true, "sampleCount": 753, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 753, "peakUsedMb": 30770, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 86138, "totalOutputTokens": 36981, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 86046, "todayTokensCached": 79713, "todayTokensOut": 9395, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 86046, "totalTokensCached": 79713, "totalTokensOut": 9395, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8213, "cachedInputTokens": 7747, "outputTokens": 214, "cost": 0, "durationMs": 141356, "outputTokensPerSec": 9.008208126993912, "at": "2026-11-05T17:45:53.151Z" }, "lastUpdated": "2026-11-05T17:45:53.151Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T17:45:55.245Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4485675, "meanTokensPerSec": 8.87, "genTokensPerSec": 8.87, "genTokens": 36981, "promptTokensPerSec": 378.94, "peakRssMb": 14189, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 86046, "output": 9395 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T16:31:03.221Z", "finishedAt": "2026-11-05T17:46:00.563Z", "timeToFirstArtifactMs": 325556, "timeToLastArtifactWriteMs": 4343114, "timeToFirstTokenMs": 33159, "firstTurnTtftMs": 19863, "timeToFirstToolCallMs": 60726 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 6, "activationEvents": 5, "completed": 4, "gateApprovals": 4, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T16:31:10.892Z", "completedAt": "2026-11-05T16:36:28.978Z", "ms": 318086, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "4c52fc3e-ce05-4ee0-a701-61dcde11a0b7" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "build", "activatedAt": "2026-11-05T17:07:50.476Z", "completedAt": "2026-11-05T17:27:02.891Z", "ms": 1152415, "gateRejections": 0, "toolCalls": 19, "sessionIds": [ "7368d88e-b4df-4e0e-a62c-18614dad9c96", "4e6de60d-3ed6-4a1a-90ad-2ab4821acf83" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T17:27:02.893Z", "completedAt": "2026-11-05T17:43:26.428Z", "ms": 983535, "gateRejections": 1, "toolCalls": 14, "sessionIds": [ "7df37340-bfc4-4cee-bdc2-35442932bc06", "facbc00d-6516-47ef-987f-4f3f86294342" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T17:43:26.430Z", "completedAt": "2026-11-05T17:45:53.107Z", "ms": 146677, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "df8d7677-5b24-4d0b-ba1d-b87a922bf011" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T16:31:09.036Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T16:31:08.937Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 650810.5 }, "sessions": { "total": 6, "taskScoped": 6, "byGezel": { "matej": 2, "mats": 1, "yerin": 3 }, "perTask": { "spreadsheet-financial-model-workflow-eval/1": 6 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 1, "maxMessages": 2, "maxContextFill": 0.1291961669921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1291961669921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "scenarioId": "craftbook-spreadsheet-model", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "artifactKind": "markdown", "artifactBytes": 3183, "artifactSha256": "ebb486d759756e80e34c83e237d250a191c18430d4104ca9b97cd278f6ebb0e8", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/sp/spreadsheet-model/versions/1.0.5/test.json rubric mapped to judge.axes by evals/src/craftbooks/scenario.ts", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": { "workflow fidelity": 7, "usefulness": 6 }, "meanScore": 6.5, "justification": "The scope has source-faithful inputs, explicit assumptions and correct hand calculations. The real workflow completed and repaired projected subtotals and growth. The calculator works, but its scenario Actual cells remain $0 and falsely show mismatch; the all-PASS review missed this visible defect. Mobile and input-validation gaps remain.", "reviewBasis": "Read the complete unchanged scope, all source fixtures, final HTML, final verdict, task notes and repair trace. Compared the earlier snapshot and reran offline browser interactions on the finalized HTML; visually checked the final desktop and earlier unchanged mobile layout. The numeric rubric targets scope/workflow; HTML is supporting evidence.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "bytes": 3183, "sha256": "ebb486d759756e80e34c83e237d250a191c18430d4104ca9b97cd278f6ebb0e8" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/workspace/spreadsheet-financial-model-workflow-eval/index.html", "bytes": 13816, "sha256": "579f5b28ef2ba3f1c3a5da0593a3255c6584e3e53389375f4632687a76c93fe6", "status": "reviewedSupportingArtifact" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/verdict.md", "bytes": 1052, "sha256": "7ea8013fd601e872633e5849822662aaee22110936a5713435858de46a3806ed", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:31.155Z", "reviewId": "059", "cellId": "r2-saluki-cuda-productivity-craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "scenarioId": "craftbook-spreadsheet-model", "status": "scored", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-spreadsheet-model", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "7fd3cd99fcfe2a9dd73bb755432b3948929aa3572099a2efb3af440657eaf911", "evidenceSha256": "9b402432c211bc467fd3fec49f32c3ed857d5aa90f7c0bd067a821d45028cf56", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "bytes": 3183, "sha256": "ebb486d759756e80e34c83e237d250a191c18430d4104ca9b97cd278f6ebb0e8", "shownBytes": 3183, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\brief.md", "bytes": 791, "sha256": "9b7893b96d2e07d87b152dcdef1c8e16395615b8bc827a34cc10dd0706ab0bda", "shownBytes": 791, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\craftbook-eval-brief.md", "bytes": 2268, "sha256": "6cebbc3d1462d96e4ee669842d5d1d5fcc2c9e0c95bf6e723aefa688b297cbdd", "shownBytes": 2268, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\records.csv", "bytes": 141, "sha256": "430befe636c28a4b2751e842a79773440af0bd7a5f1b742ea177c6ef204d74c6", "shownBytes": 141, "truncated": false } ], "scoreAxes": { "workflow fidelity": 7, "usefulness": 7 }, "justification": "scope.md is at the gated artifact path. It locks purpose, drivers A1–A4, calculation logic, periods and an acceptance checklist. Figures match records.csv: revenue 4,760, default churn loss 835, projected 3,925, and the high-tier scenario 2,660. Minor flaws: \"Net revenue after churn\" duplicates the projected line, and \"revenue at risk\" is only high-tier revenue.", "limitations": "Only scope.md was reviewed. No evidence of advance_task_step, a task-note, or the build, evaluate and finish steps. The brief's returns-desk facts are not used, though records.csv grounds the model. No runtime gate results were visible.", "meanScore": 7, "durationMs": 16688, "costUsd": 0.028699799999999998, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5447, "cache_read_input_tokens": 778, "output_tokens": 683, "output_tokens_details": { "thinking_tokens": 322 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5447, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 683, "cache_read_input_tokens": 778, "cache_creation_input_tokens": 5447, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5447 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 683, "cacheReadInputTokens": 778, "cacheCreationInputTokens": 5447, "webSearchRequests": 0, "costUSD": 0.028699799999999998, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 322, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5447, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-spreadsheet-model / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 74m 57s; budget used: 100.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 325556`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3148,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 0 / 10 | The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 39`; `facts.timing.timeToFirstArtifactMs: 325556`; `facts.outcome.durationMs: 4497342` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 0 + 0.15 * 10 = **8.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14189 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30770 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.87 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 86138 + 36981 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 6 → 4 | `continuity.steps` |\n| Gate rejections / redrives | 1 / 0 | `continuity.steps` |\n| Median step wall-clock | 651s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 6 / 1.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 13% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 3 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 325556`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3148,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_BOTH_SEVERE\n\nThe trial exhausted its budget while making at least thirty tool calls.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 39`\n- `facts.timing.timeToFirstArtifactMs: 325556`\n- `facts.outcome.durationMs: 4497342`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 325556`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3148,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}`\n- Efficiency: The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 39`; `facts.timing.timeToFirstArtifactMs: 325556`; `facts.outcome.durationMs: 4497342`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 8/10** is unchanged. Declared advisory axes: workflow fidelity 7/10; usefulness 6/10. Mean **6.50/10**, codex-inline/current-session.\n\nThe scope has source-faithful inputs, explicit assumptions and correct hand calculations. The real workflow completed and repaired projected subtotals and growth. The calculator works, but its scenario Actual cells remain $0 and falsely show mismatch; the all-PASS review missed this visible defect. Mobile and input-validation gaps remain.\n\nRead the complete unchanged scope, all source fixtures, final HTML, final verdict, task notes and repair trace. Compared the earlier snapshot and reran offline browser interactions on the finalized HTML; visually checked the final desktop and earlier unchanged mobile layout. The numeric rubric targets scope/workflow; HTML is supporting evidence.\n\n## Positives\n\n- The real craftbook completed scope, build, evaluate and finish, with four gate approvals and one rejection/loop. This is actual terminal workflow progress, not only a manually assigned completion label (outcome.success; continuity.steps.completed:4, gateApprovals:4, gateRejections:1; log.txt terminal milestones).\n- Scope arithmetic is correct: source revenue4760, expected loss835, default projected3925, no-churn4760 and high-only100%2660. Assumed churn rates are explicitly labeled, and all four source records match the fixture (sniff.latest 6/6; scope.md versus records.csv; independent browser audit).\n- The repair adds growth to per-customer projections and makes their sum agree with the projected total. At10% growth the four rows sum4317.50, matching the primary output. Scenario clicks correctly select inputs and recompute the primary value (toolUse.byTool.replace_lines:1, replace_in_file:2; advisory-browser-final/audit.json).\n\n## Negatives\n\n- All three scenario Actual cells remain0 and display mismatch, before and after clicks. render() reads those cells but never populates them. The final all-PASS verdict misses the visible failure even though the core calculation is correct (artifacts.htmlFiles; final verdict; browser audit scenarioRows).\n- All three validate calls pass syntax/structure while explicitly skipping runtime-load as browser-runtime-not-installed. Hand-checking formulas and reading event binding are useful but do not establish a working rendered self-check. The recorder also fails its long file-URL screenshot; the separate offline audit succeeds with setContent (toolUse.byTool.validate:3; saved validation responses; log.txt).\n- The review gate loops on the literal FAIL in the phrase not a FAIL, even though the reviewer marked every criterion PASS. That false lexical rejection prompted useful subtotal repair, but it was not reliable detection of the actual scenario-display defect. A previous verdict write was rejected for correctly derived arithmetic until sources were reread (continuity.steps.gateRejections:1; recording/task-notes.json; advisory-tool-evidence.json).\n- The task takes74m57s,39calls and36,981 recorded output tokens. At360px width the page is513px wide; input labels are not programmatically associated, scenario rows lack keyboard controls, and an out-of-range200% value still computes without visible validation feedback (outcome.durationMs; toolUse.totalToolCalls; perf.derived; browser audit).\n\n## Performance and GPU fit\n\nElapsed 74.96 minutes; 36981 recorded generated tokens at **8.87 tok/s**, prefill 378.94 tok/s. Peak VRAM **10630/12227 MiB**, process-tree RSS **14189 MiB**, whole-system RAM **30770/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications; shared GPU memory and paging were not measured.\n\nNative0.1.57 CUDA, one normal launch, 65,536-token allocation, one slot, f16KV and reasoning budget4096. Dense FFNs for39/64 layers on GPU and25 on CPU. No failed endpoint, explicit OOM or context overflow was found; recorded exit is expected SIGTERM on trial shutdown. Final captured maximum session fill 12.9% is a session snapshot, not a full peak-KV measurement. No mid-turn compactions. Performance remains separate from fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nIn the evaluate prompt for ../gilde/data/craftbook-templates/sp/spreadsheet-model/versions/1.0.5/craftbook.json, propose one explicit observable check: after changing a driver, inspect both the primary output and every displayed verification result; distinguish formula inspection from actual UI execution. Keep current content frozen. Validate on this calculator and an unrelated interactive form before generalizing to shared review instructions.\n\n## Strategic experiment — not applied\n\nIn packages/mcp/src/validate.ts, expose runtime availability and actual browser execution as separate statuses that a static PASS cannot obscure. For verdict gates, replace raw forbidden-word scanning with structured per-criterion pass/fail results in packages/service/src/tasks/gate-eval.ts and step-gate.ts; keep prose notes independent. Measure false rejection of negated FAIL and false acceptance of broken self-check displays across calculator and non-financial UI tasks.\n\n## Generalization and next experiment\n\nThe first repetition failed fixed4.5/advisory4 with incorrect scope math and no HTML. This repetition reaches fixed8/advisory6.5, with correct arithmetic and an executable calculator. No tuning changed; this is repetition variability, not proof of a native-version effect. Tankcombat also passed static gates despite broken interactions, making browser-observed behavior a cross-domain issue.\n\nEvaluate any proposal separately against the indicated unrelated task before recommending it. No source, tuning, prompt, gate, model or campaign-state change was made, and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/facts.json): fail, fixed4.5, 77.74 minutes, 2.73tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/facts.json): pass, fixed9.5, 2.46 minutes, 128.8tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-32-13-163Z-94l9/facts.json): pass, fixed9.5, 2.46 minutes, 125.02tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are contextual; no causal native-version improvement is inferred.\n\n## Strategic note\n\nWorkflow completion and correct primary arithmetic are established. Complete QA is not: the displayed sanity checks remain broken despite the all-PASS verdict.\n\n## Raw evidence\n\nSee advisory-browser-final/audit.json and desktop.png, the earlier advisory-progress-2026-11-05T1655 snapshot, recording/task-notes.json, final verdict and advisory-tool-evidence.json. Also see facts.json, score.json, historical-comparison.json, advisory-artifact-review.json and daemon.log.\n" }, { "cellId": "r2-saluki-cuda-productivity-conflict-synthesis", "baselineCellId": "r2-saluki-cuda-productivity-conflict-synthesis", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T17:48:58.019Z", "finishedAt": "2026-11-05T17:56:09.942Z", "durationMs": 431923, "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 341998 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 2279, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.10918053489799405 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 341998 }, { "source": "facts", "path": "outcome.durationMs", "value": 431923 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 72, "peakRssMb": 9792, "peakCpuPercent": 1206.8, "firstAtMs": 1, "lastAtMs": 419577 }, "gpu": { "available": true, "sampleCount": 72, "peakUtilPercent": 96, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 72, "peakUsedMb": 27257, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12653, "totalOutputTokens": 3333, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12641, "todayTokensCached": 11808, "todayTokensOut": 551, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12641, "totalTokensCached": 11808, "totalTokensOut": 551, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 12641, "cachedInputTokens": 11808, "outputTokens": 551, "cost": 0, "durationMs": 413749, "outputTokensPerSec": 8.749589485169702, "at": "2026-11-05T17:56:02.195Z" }, "lastUpdated": "2026-11-05T17:56:02.195Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T17:56:05.134Z" } }, "source": "engine-log" }, "derived": { "durationMs": 420907, "meanTokensPerSec": 8.76, "genTokensPerSec": 8.76, "genTokens": 3333, "promptTokensPerSec": 552.72, "peakRssMb": 9792, "peakGpuUtilPercent": 96, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 12641, "output": 551 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T17:49:03.797Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T17:49:03.695Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.142425537109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.142425537109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 431923, "timeoutMs": 3956044, "budgetUsedFraction": 0.10918053489799405 }, "timing": { "startedAt": "2026-11-05T17:48:58.019Z", "finishedAt": "2026-11-05T17:56:09.942Z", "timeToFirstArtifactMs": 341998, "timeToLastArtifactWriteMs": 424177, "timeToFirstTokenMs": 36344, "firstTurnTtftMs": 25917, "timeToFirstToolCallMs": 46892 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 3, "byTool": { "read_files": 1, "write_file": 1, "read_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 343341, "filePath": "conflict-synthesis", "bytes": 2279, "score": 12, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "citations" ], "failReason": null }, { "atMs": 426564, "filePath": "conflict-synthesis", "bytes": 2279, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } ], "latest": { "filePath": "conflict-synthesis", "bytes": 2279, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T17:48:58.022Z] [trial] id=conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw scenario=conflict-synthesis model=underdog-saluki-27b-iq2-mix", "[2026-11-05T17:48:58.022Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T17:48:58.022Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "[2026-11-05T17:48:58.022Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T17:48:58.022Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-WJoRPz", "[2026-11-05T17:48:58.025Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T17:48:58.030Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T17:48:58.035Z] [trial] throughput-scaled ceiling: 30m → 66m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T17:48:58.035Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-WJoRPz", "[2026-11-05T17:49:03.754Z] [trial] daemon spawned pid=19204 port=https://127.0.0.1:63087", "[2026-11-05T17:49:05.051Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T17:49:05.067Z] [trial] meester=zara", "[2026-11-05T17:49:05.067Z] [trial] running scenario setup hook", "[2026-11-05T17:49:05.475Z] [trial] scenario setup complete", "[2026-11-05T17:49:05.476Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T17:49:05.552Z] [poll] initial digests hard=bc0cc0e903061966 soft=eb09f6ff71d138b4", "[2026-11-05T17:49:05.623Z] [poll] hard-progress: bc0cc0e903061966 → 1969cdc440c8a8b9 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T17:49:46.055Z] [poll] hard-progress: 1969cdc440c8a8b9 → 4226db5ae7d1e9ce after 40s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=37,stream=111) sniff=0/0B)", "[2026-11-05T17:54:41.421Z] [poll] hard-progress: 4226db5ae7d1e9ce → 6bcd918c9b13a1ef after 295s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=879,stream=3243) sniff=12/2279B)", "[2026-11-05T17:54:59.238Z] [poll] hard-progress: 6bcd918c9b13a1ef → 91594c2fd950b2f5 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=918,stream=3356) sniff=12/2279B)", "[2026-11-05T17:56:04.583Z] [poll] terminal: success=true reason=all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "[2026-11-05T17:56:05.154Z] [capture] dumped 1 session(s)", "[2026-11-05T17:56:05.157Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-05T17:56:05.211Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T17:56:09.941Z] [trial] daemon shut down" ] }, "metrics": { "process": { "sampleCount": 72, "peakRssMb": 9792, "peakCpuPercent": 1206.8, "firstAtMs": 1, "lastAtMs": 419577 }, "gpu": { "available": true, "sampleCount": 72, "peakUtilPercent": 96, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 72, "peakUsedMb": 27257, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12653, "totalOutputTokens": 3333, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 12641, "todayTokensCached": 11808, "todayTokensOut": 551, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 12641, "totalTokensCached": 11808, "totalTokensOut": 551, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 12641, "cachedInputTokens": 11808, "outputTokens": 551, "cost": 0, "durationMs": 413749, "outputTokensPerSec": 8.749589485169702, "at": "2026-11-05T17:56:02.195Z" }, "lastUpdated": "2026-11-05T17:56:02.195Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T17:56:05.134Z" } }, "source": "engine-log" }, "derived": { "durationMs": 420907, "meanTokensPerSec": 8.76, "genTokensPerSec": 8.76, "genTokens": 3333, "promptTokensPerSec": 552.72, "peakRssMb": 9792, "peakGpuUtilPercent": 96, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 12641, "output": 551 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T17:48:58.019Z", "finishedAt": "2026-11-05T17:56:09.942Z", "timeToFirstArtifactMs": 341998, "timeToLastArtifactWriteMs": 424177, "timeToFirstTokenMs": 36344, "firstTurnTtftMs": 25917, "timeToFirstToolCallMs": 46892 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T17:49:03.797Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T17:49:03.695Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.142425537109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.142425537109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "scenarioId": "conflict-synthesis", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/workspace/skylark-launch-synthesis/synthesis.md", "artifactKind": "markdown", "artifactBytes": 2297, "artifactSha256": "0261efd8d80b109d110f652c0969a0c9b5dfd8bf16d2412548291379c0c9d456", "axisSource": "evals/src/scenarios/conflict-synthesis.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The synthesis correctly selects September1, EUR210,000 and Marcus with source precedence and citations. It invents a contractor-count ambiguity from one aggregate finance row, and states the historical Tuesday cadence as current while also asking if it remains valid. Canonical conflicts are correctly resolved.", "reviewBasis": "Read the full final synthesis and all six source/style files. Checked canonical facts, line-item sum, historical-value placement and whether open questions reflect real source disagreement. Inspected the batch-read/write/read trace.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/workspace/skylark-launch-synthesis/synthesis.md", "bytes": 2297, "sha256": "0261efd8d80b109d110f652c0969a0c9b5dfd8bf16d2412548291379c0c9d456" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:29.929Z", "reviewId": "060", "cellId": "r2-saluki-cuda-productivity-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "scenarioId": "conflict-synthesis", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "baselineCellId": "r2-saluki-cuda-productivity-conflict-synthesis", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "70720e7e0845d7b2ebf93ac32bd6d41dd9f2c6a91b11563c2e85bbb0d839eb7d", "evidenceSha256": "774eabb46e716ca081d3cfeb35fcfdfed109a7698ec242bbb5880d5d04dc8272", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/workspace/skylark-launch-synthesis/synthesis.md", "bytes": 2297, "sha256": "0261efd8d80b109d110f652c0969a0c9b5dfd8bf16d2412548291379c0c9d456", "shownBytes": 2297, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "synthesis.md has the four required sections in order. It resolves three conflicts (date, budget, DRI), each quoting both values and file names with a winner and reason. The plan uses only winning values and cites backticked files well over six times. Superseded values appear only in the conflicts section.", "limitations": "The source files were not shown, so I could not check completeness against all conflicts, such as a possible third-party or other hidden conflict. The Overview's \"two contractors\" and onboarding scope claims were not verified. No numeric axes were declared, so no scoring was done.", "meanScore": null, "durationMs": 14530, "costUsd": 0.013454, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2605, "cache_read_input_tokens": 700, "output_tokens": 296, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2605, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 296, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2605, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2605 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 296, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2605, "webSearchRequests": 0, "costUSD": 0.013454, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2605, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - conflict-synthesis / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 7m 12s; budget used: 10.9%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 341998`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2279,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.10918053489799405`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 341998`; `facts.outcome.durationMs: 431923` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9792 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 96% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27257 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.76 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 12653 + 3333 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 14% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 341998`\n- `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2279,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.10918053489799405`\n- `facts.toolUse.totalToolCalls: 3`\n- `facts.timing.timeToFirstArtifactMs: 341998`\n- `facts.outcome.durationMs: 431923`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 341998`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2279,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.10918053489799405`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 341998`; `facts.outcome.durationMs: 431923`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is unchanged. No declared axes; prose review only.\n\nNo declared axes. The synthesis correctly selects September1, EUR210,000 and Marcus with source precedence and citations. It invents a contractor-count ambiguity from one aggregate finance row, and states the historical Tuesday cadence as current while also asking if it remains valid. Canonical conflicts are correctly resolved.\n\nRead the full final synthesis and all six source/style files. Checked canonical facts, line-item sum, historical-value placement and whether open questions reflect real source disagreement. Inspected the batch-read/write/read trace.\n\n## Positives\n\n- All13 signals pass. Engineering supersedes product on launch timing, finance controls budget, and org.md controls ownership. Old exact values remain confined to Conflicts and resolutions (outcome.success; sniff.latest 13/13; final synthesis versus six source files).\n- The three line items sum210000, current owners and onboarding scope match sources, and the output cites the relevant filenames. One batch source read and one write avoid the repeated repair cycle seen in the core copy (toolUse.byTool.read_files:1,write_file:1,read_file:1; final artifact).\n\n## Negatives\n\n- One aggregate contractors finance row does not imply one contractor. The open question Is the contractor count1or2 creates a conflict where product explicitly says two and finance is silent on headcount. Calling an EUR amount a dollar figure adds a currency-label slip (sniff.latest claims-source-supported covers narrower facts; final Open questions versus memo-product.md and finance.csv).\n- The consolidated plan states Tuesday syncs as current because no newer source overrides them, but the same document asks whether that historical cadence is still in effect. Marking the cadence provisional would better reflect uncertainty (artifacts.otherFileCount; memo-oldplan.md and final plan/open-question pair).\n\n## Performance and GPU fit\n\nElapsed 7.20 minutes; 3333 recorded generated tokens at **8.76 tok/s**, prefill 552.72 tok/s. Peak VRAM **10626/12227 MiB**, process-tree RSS **9792 MiB**, whole-system RAM **27257/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications; shared GPU memory and paging were not measured.\n\nNative0.1.57 CUDA, one normal launch, 65,536-token allocation, one slot, f16KV and reasoning budget4096. Dense FFNs for39/64 layers on GPU and25 on CPU. No failed endpoint, explicit OOM or context overflow was found; recorded exit is expected SIGTERM on trial shutdown. Final captured maximum session fill 14.2% is a session snapshot, not a full peak-KV measurement. No mid-turn compactions. Performance remains separate from fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nIn a proposed generic packages/core/src/model-profile/behaviors/prompt-evidence-conflicts.ts, distinguish conflicting claims from differences in granularity: one aggregate row does not establish entity count, and unconfirmed historical facts should retain an explicit status. A/B on memo synthesis and an unrelated inventory summary; track invented ambiguity and unnecessary clarification.\n\n## Strategic experiment — not applied\n\nIn packages/mcp/src/validate.ts, explore advisory provenance for claim value, unit, granularity and confirmation status. Cross-check prose against its own open questions, so a fact is not simultaneously treated as settled and awaiting confirmation. Keep the current exact-value gates unchanged.\n\n## Generalization and next experiment\n\nSame-campaign conflict scores are8.5,10,8.5,10 across the core/productivity copies. This run uses3calls without native recovery, while the second core copy used30calls and recovery. Main source precedence is stable; unsupported interpretation remains variable. The meeting follow-up similarly preserves its register while strengthening a scheduling commitment in prose.\n\nEvaluate any proposal separately against the indicated unrelated task before recommending it. No source, tuning, prompt, gate, model or campaign-state change was made, and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/facts.json): pass, fixed8.5, 18.84 minutes, 8.55tok/s.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/facts.json): pass, fixed10, 28.92 minutes, 2.7tok/s.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/facts.json): pass, fixed8.5, 55.01 minutes, 2.84tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are contextual; no causal native-version improvement is inferred.\n\n## Strategic note\n\nCorrect canonical reconciliation does not rule out invented open questions or inconsistent confidence. Treat those as separate advisory findings.\n\n## Raw evidence\n\nSee final synthesis, memo-product.md, memo-engineering.md, memo-oldplan.md, org.md, finance.csv, style.md and advisory-tool-evidence.json. Also see facts.json, score.json, historical-comparison.json, advisory-artifact-review.json and daemon.log.\n" }, { "cellId": "r2-saluki-cuda-productivity-docblocks-theme-roundtrip", "baselineCellId": "r2-saluki-cuda-productivity-docblocks-theme-roundtrip", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T17:58:27.610Z", "finishedAt": "2026-11-05T18:04:04.217Z", "durationMs": 336607, "success": true, "reason": "theme report is tool-grounded and the saved deck carries the brand (ordered-sections, theme-id, fonts, palette, slide-count, compare-cited, differences-not-whitewashed, recommendation, docblocks-provenance)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 269856 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 1834, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.0850867684990359 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 269856 }, { "source": "facts", "path": "outcome.durationMs", "value": 336607 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 55, "peakRssMb": 12450, "peakCpuPercent": 1207.5, "firstAtMs": 0, "lastAtMs": 319308 }, "gpu": { "available": true, "sampleCount": 55, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 55, "peakUsedMb": 28774, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 33549, "totalOutputTokens": 2020, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 27466, "todayTokensCached": 27443, "todayTokensOut": 467, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 27466, "totalTokensCached": 27443, "totalTokensOut": 467, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 27466, "cachedInputTokens": 27443, "outputTokens": 467, "cost": 0, "durationMs": 311477, "outputTokensPerSec": 8.820188662321293, "at": "2026-11-05T18:03:51.920Z" }, "lastUpdated": "2026-11-05T18:03:51.920Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T18:03:53.856Z" } }, "source": "engine-log" }, "derived": { "durationMs": 320579, "meanTokensPerSec": 8.94, "genTokensPerSec": 8.94, "genTokens": 2020, "promptTokensPerSec": 485.3, "peakRssMb": 12450, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 27466, "output": 467 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T17:58:33.228Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T17:58:33.130Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wieke": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.2438812255859375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2438812255859375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "theme report is tool-grounded and the saved deck carries the brand (ordered-sections, theme-id, fonts, palette, slide-count, compare-cited, differences-not-whitewashed, recommendation, docblocks-provenance)", "durationMs": 336607, "timeoutMs": 3956044, "budgetUsedFraction": 0.0850867684990359 }, "timing": { "startedAt": "2026-11-05T17:58:27.610Z", "finishedAt": "2026-11-05T18:04:04.217Z", "timeToFirstArtifactMs": 269856, "timeToLastArtifactWriteMs": 324311, "timeToFirstTokenMs": 56378, "firstTurnTtftMs": 43545, "timeToFirstToolCallMs": 60252 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "infer_theme_from_file": 1, "apply_inferred_theme": 1, "convert_document": 1, "save_artifact": 1, "inspect_document": 1, "compare_documents": 1, "write_file": 2, "read_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 325738, "filePath": "docblocks-theme", "bytes": 1834, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } ], "latest": { "filePath": "docblocks-theme", "bytes": 1834, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T17:58:27.613Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "[2026-11-05T17:58:27.613Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T17:58:27.613Z] [trial] id=docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux scenario=docblocks-theme-roundtrip model=underdog-saluki-27b-iq2-mix", "[2026-11-05T17:58:27.613Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-docblocks-theme-roundtrip-7oTHHj", "[2026-11-05T17:58:27.613Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T17:58:27.615Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T17:58:27.619Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T17:58:27.624Z] [trial] throughput-scaled ceiling: 30m → 66m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T17:58:27.624Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-docblocks-theme-roundtrip-7oTHHj", "[2026-11-05T17:58:33.233Z] [trial] daemon spawned pid=23532 port=https://127.0.0.1:64884", "[2026-11-05T17:58:34.090Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T17:58:34.107Z] [trial] meester=orion", "[2026-11-05T17:58:34.107Z] [trial] running scenario setup hook", "[2026-11-05T17:58:34.512Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T17:58:34.510Z] [trial] scenario setup complete", "[2026-11-05T17:58:34.586Z] [poll] initial digests hard=eedbd087764ec531 soft=0ad9b1c6d4b62363", "[2026-11-05T17:58:34.671Z] [poll] hard-progress: eedbd087764ec531 → 8116be37426c1158 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T17:59:33.716Z] [poll] hard-progress: 8116be37426c1158 → 588ed301cd587b3a after 59s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=34,stream=84) sniff=0/0B)", "[2026-11-05T17:59:39.508Z] [poll] hard-progress: 588ed301cd587b3a → 24f6743f9e891c16 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=43,stream=125) sniff=0/0B)", "[2026-11-05T17:59:57.243Z] [poll] hard-progress: 24f6743f9e891c16 → c03ef8b3476eeb37 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=77,stream=296) sniff=0/0B)", "[2026-11-05T18:00:09.058Z] [poll] hard-progress: c03ef8b3476eeb37 → d96aee6c4d5f83d7 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=99,stream=406) sniff=0/0B)", "[2026-11-05T18:00:20.890Z] [poll] hard-progress: d96aee6c4d5f83d7 → 0605478f6e05bb70 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=120,stream=514) sniff=0/0B)", "[2026-11-05T18:00:32.513Z] [poll] hard-progress: 0605478f6e05bb70 → 07433466721e6f33 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=145,stream=655) sniff=0/0B)", "[2026-11-05T18:01:37.547Z] [poll] hard-progress: 07433466721e6f33 → 77620f896c8c5bd5 after 65s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=310,stream=1593) sniff=0/0B)", "[2026-11-05T18:02:01.194Z] [poll] hard-progress: 77620f896c8c5bd5 → 07e2e04b4ea10cb7 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=326,stream=1620) sniff=0/0B)" ] }, "metrics": { "process": { "sampleCount": 55, "peakRssMb": 12450, "peakCpuPercent": 1207.5, "firstAtMs": 0, "lastAtMs": 319308 }, "gpu": { "available": true, "sampleCount": 55, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 55, "peakUsedMb": 28774, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 33549, "totalOutputTokens": 2020, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 27466, "todayTokensCached": 27443, "todayTokensOut": 467, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 27466, "totalTokensCached": 27443, "totalTokensOut": 467, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 27466, "cachedInputTokens": 27443, "outputTokens": 467, "cost": 0, "durationMs": 311477, "outputTokensPerSec": 8.820188662321293, "at": "2026-11-05T18:03:51.920Z" }, "lastUpdated": "2026-11-05T18:03:51.920Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T18:03:53.856Z" } }, "source": "engine-log" }, "derived": { "durationMs": 320579, "meanTokensPerSec": 8.94, "genTokensPerSec": 8.94, "genTokens": 2020, "promptTokensPerSec": 485.3, "peakRssMb": 12450, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 27466, "output": 467 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T17:58:27.610Z", "finishedAt": "2026-11-05T18:04:04.217Z", "timeToFirstArtifactMs": 269856, "timeToLastArtifactWriteMs": 324311, "timeToFirstTokenMs": 56378, "firstTurnTtftMs": 43545, "timeToFirstToolCallMs": 60252 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T17:58:33.228Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T17:58:33.130Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wieke": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.2438812255859375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2438812255859375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "scenarioId": "docblocks-theme-roundtrip", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/workspace/brand-theme-alignment/theme-report.md", "artifactKind": "markdown", "artifactBytes": 1840, "artifactSha256": "20bb01a7cdb3acbbce4e4a8044a36fbd21397a3863b1bbb8eef8691fe27fa1ce", "axisSource": "Declared judge.axes in evals/src/scenarios/docblocks-theme-roundtrip.ts", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ], "scoreAxes": { "grounding": 5, "candor": 5, "usefulness": 5 }, "meanScore": 5, "justification": "Theme values, four slides and comparison figures match the tool receipts and saved PPTX. The report asserts that26% text similarity is expected because copy was reworked, although inspected slide text preserves the source. It lists accessibility/timing degradation but the recommendation largely smooths those findings over.", "reviewBasis": "Read the complete report, source Markdown and all six successful DocBlocks receipts. Inspected the saved PPTX ZIP: four slides, required brand fonts and source-faithful slide text; SHA matches save_artifact. No deck visual-layout claim is made from ZIP inspection.", "deliverables": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/workspace/brand-theme-alignment/theme-report.md", "bytes": 1840, "sha256": "20bb01a7cdb3acbbce4e4a8044a36fbd21397a3863b1bbb8eef8691fe27fa1ce" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/artifacts/brand-theme-alignment/quarterly-review.pptx", "bytes": 19853, "sha256": "f21a9a27f395dddaa5ca86666c663f98dd85998477f23f913e67ecf40aceb42c", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:42.385Z", "reviewId": "061", "cellId": "r2-saluki-cuda-productivity-docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "scenarioId": "docblocks-theme-roundtrip", "status": "scored", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "baselineCellId": "r2-saluki-cuda-productivity-docblocks-theme-roundtrip", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "194e0189ba5b2ef36711c9d946caee0bb7d244ef2baa1623ed795266991c2f75", "evidenceSha256": "e9d2ef77aa030a41e4e53c648cef02ef638965b10142df12b9c8998d875d82f8", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/workspace/brand-theme-alignment/theme-report.md", "bytes": 1840, "sha256": "20bb01a7cdb3acbbce4e4a8044a36fbd21397a3863b1bbb8eef8691fe27fa1ce", "shownBytes": 1840, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux\\workspace\\brand-theme-alignment\\source\\quarterly-review.md", "bytes": 443, "sha256": "70a119718fdfd89aba5d0dd2bdc2e7df8fb45f749afc71d56bb5980e1fb030a8", "shownBytes": 443, "truncated": false } ], "scoreAxes": { "grounding": 5, "candor": 6, "usefulness": 4 }, "justification": "Required theme values match. But it adds a \"Halvard Brand\" theme name and \"sans-serif fallback\", neither on the allowed list. It also gives untooled explanations (\"restructured\", \"re-flowed\", \"as expected\"). It says \"end-to-end\" while theme similarity was 0%. Candor: it lists every score, but explains away the 26% text score. Recommendation is vague.", "limitations": "Cannot verify whether \"Halvard Brand\" or fallback details appeared in tool output, since the tool responses are not shown. The slide count and similarity figures are not checked here; the gate covers them. Source-only review.", "meanScore": 5, "durationMs": 17876, "costUsd": 0.0208324, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3147, "cache_read_input_tokens": 804, "output_tokens": 816, "output_tokens_details": { "thinking_tokens": 481 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3147, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 816, "cache_read_input_tokens": 804, "cache_creation_input_tokens": 3147, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3147 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 816, "cacheReadInputTokens": 804, "cacheCreationInputTokens": 3147, "webSearchRequests": 0, "costUSD": 0.0208324, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 481, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3147, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - docblocks-theme-roundtrip / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 5m 37s; budget used: 8.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 269856`; `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1834,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.0850867684990359`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 269856`; `facts.outcome.durationMs: 336607` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12450 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10632 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28774 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.94 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 33549 + 2020 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 24% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 269856`\n- `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1834,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.0850867684990359`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 269856`\n- `facts.outcome.durationMs: 336607`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 269856`; `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1834,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.0850867684990359`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 269856`; `facts.outcome.durationMs: 336607`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is unchanged. Declared advisory axes: grounding 5/10; candor 5/10; usefulness 5/10. Mean **5.00/10**, codex-inline/current-session.\n\nTheme values, four slides and comparison figures match the tool receipts and saved PPTX. The report asserts that26% text similarity is expected because copy was reworked, although inspected slide text preserves the source. It lists accessibility/timing degradation but the recommendation largely smooths those findings over.\n\nRead the complete report, source Markdown and all six successful DocBlocks receipts. Inspected the saved PPTX ZIP: four slides, required brand fonts and source-faithful slide text; SHA matches save_artifact. No deck visual-layout claim is made from ZIP inspection.\n\n## Positives\n\n- All nine fixed signals pass and every required DocBlocks operation succeeded: infer, apply, convert, save, inspect and compare. The saved19853-byte PPTX matches the tool SHA and contains four real slide parts with Kelder Grotesk/Marne Text (outcome.success; sniff.latest.docblocks-provenance; toolUse.byTool; advisory-pptx-inspection.json).\n- The report retains the exact theme id, fonts, primary/background colors, slide count and all comparison percentages, including the26% text,84% timing and64% accessibility figures (sniff.latest theme-id/fonts/palette/slide-count/compare-cited; tool receipts versus final Markdown).\n\n## Negatives\n\n- The claimed explanation for low text similarity is unsupported. Extracted slide text retains the source wording; the comparison reports a number and degradation status, not that substantive rewriting caused it or that26% is expected. The report should label uncertainty instead of normalizing the metric (sniff.latest passes numeric provenance; inspect_document result and saved slide XML versus final prose).\n- The recommendation emphasizes100% categories and makes text review optional, without a concrete response to timing/accessibility differences. The counts behind those metrics also matter: accessibility features increase7→11 even though the tool labels similarity degraded. Similarity is not itself proof of lost accessibility quality (toolUse.byTool.compare_documents:1; comparison metrics; final recommendation).\n- A100% table/media similarity is vacuous here: both source and result have zero tables/assets. Calling those preserved assets without context overstates what the successful round trip demonstrates (artifacts.otherFileCount; inspect_document and compare_documents receipts).\n\n## Performance and GPU fit\n\nElapsed 5.61 minutes; 2020 recorded generated tokens at **8.94 tok/s**, prefill 485.3 tok/s. Peak VRAM **10632/12227 MiB**, process-tree RSS **12450 MiB**, whole-system RAM **28774/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications; shared GPU memory and paging were not measured.\n\nNative0.1.57 CUDA, one normal launch, 65,536-token allocation, one slot, f16KV and reasoning budget4096. Dense FFNs for39/64 layers on GPU and25 on CPU. No failed endpoint, explicit OOM or context overflow was found; recorded exit is expected SIGTERM on trial shutdown. Final captured maximum session fill 24.4% is a session snapshot, not a full peak-KV measurement. No mid-turn compactions. Performance remains separate from fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nIn a proposed packages/core/src/model-profile/behaviors/prompt-tool-report-candor.ts, require reports to separate observed tool measurements, inferred causes and next verification actions. When a metric is surprising, label its cause unknown unless evidence establishes it. A/B on document conversion and an unrelated performance report; measure unsupported causal explanations without suppressing useful recommendations.\n\n## Strategic experiment — not applied\n\nIn a proposed packages/service/src/providers/mcp-wrappers/docblocks-comparison-evidence.ts adapter, expose feature counts and applicability beside similarity and retain source/result text for diagnostics. In packages/mcp/src/validate.ts or shared evidence checks, treat a numeric match as provenance for the number, not for an invented explanation. Test text-preserving conversions and actual content-loss cases before proposing a general change; do not alter this campaign.\n\n## Generalization and next experiment\n\nThe first DocBlocks repetition also passed fixed10 but advisory5.33 with unsupported interpretation and a recovered endpoint failure. This run passes fixed10/advisory5.0 with one normal native launch and no such recovery. Report candor remains the recurring issue. Meeting and conflict tasks show the same pattern of correct quoted facts followed by stronger unsupported interpretation.\n\nEvaluate any proposal separately against the indicated unrelated task before recommending it. No source, tuning, prompt, gate, model or campaign-state change was made, and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/facts.json): pass, fixed10, 19.04 minutes, 2.58tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/facts.json): fail, fixed5.9, 8.54 minutes, 115.1tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-46-29-887Z-eoao/facts.json): fail, fixed5.9, 9.17 minutes, 116.62tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are contextual; no causal native-version improvement is inferred.\n\n## Strategic note\n\nThe branded binary conversion is real and completed. The explanation of comparison differences remains less reliable than the operation and numeric provenance.\n\n## Raw evidence\n\nSee advisory-pptx-inspection.json, advisory-tool-evidence.json, source/quarterly-review.md, final theme-report.md and the saved PPTX. No visual slide QA is inferred from package inspection. Also see facts.json, score.json, historical-comparison.json, advisory-artifact-review.json and daemon.log.\n" }, { "cellId": "r2-saluki-cuda-productivity-craftbook-powerpoint-deck", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-powerpoint-deck", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T18:07:11.971Z", "finishedAt": "2026-11-05T18:39:42.532Z", "durationMs": 1950561, "success": true, "reason": "craftbook-powerpoint-deck passed 21 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 301167 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.4226215147815404 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 34 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 301167 }, { "source": "facts", "path": "outcome.durationMs", "value": 1950561 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 326, "peakRssMb": 15087, "peakCpuPercent": 1312.8, "firstAtMs": 1, "lastAtMs": 1931254 }, "gpu": { "available": true, "sampleCount": 326, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 326, "peakUsedMb": 31446, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 83369, "totalOutputTokens": 13813, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 85004, "todayTokensCached": 81409, "todayTokensOut": 2061, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 85004, "totalTokensCached": 81409, "totalTokensOut": 2061, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 36916, "cachedInputTokens": 36847, "outputTokens": 48, "cost": 0, "durationMs": 242927, "outputTokensPerSec": 8.459664320519762, "at": "2026-11-05T18:39:27.351Z" }, "lastUpdated": "2026-11-05T18:39:27.351Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T18:39:31.016Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1932639, "meanTokensPerSec": 8.55, "genTokensPerSec": 8.55, "genTokens": 13813, "promptTokensPerSec": 370.98, "peakRssMb": 15087, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 85004, "output": 2061 } } } }, "craftbook": { "craftbookId": "powerpoint-deck", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "powerpoint-grounding-eval/1", "stepId": "research", "activatedAt": "2026-11-05T18:07:19.630Z", "completedAt": "2026-11-05T18:12:13.334Z", "ms": 293704, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "57077318-c602-4166-9e2d-252e282ebf1a" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "outline", "activatedAt": "2026-11-05T18:12:13.336Z", "completedAt": "2026-11-05T18:18:18.182Z", "ms": 364846, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "818d2c63-2b6d-4ea2-b11d-63f3878ced43" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "write", "activatedAt": "2026-11-05T18:18:18.183Z", "completedAt": "2026-11-05T18:20:25.536Z", "ms": 127353, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "d55597af-1774-4e26-b01c-2b58b4bf6707" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "review", "activatedAt": "2026-11-05T18:20:25.537Z", "completedAt": "2026-11-05T18:35:10.737Z", "ms": 885200, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "9f9c1cdc-93e9-4a0d-bc61-2f6600aa4e45" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "save", "activatedAt": "2026-11-05T18:35:10.739Z", "completedAt": "2026-11-05T18:39:27.287Z", "ms": 256548, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "839ab60d-a7ff-4823-9776-5c47c9e10852" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T18:07:17.977Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T18:07:17.869Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 293704 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "clemence": 1, "elzbieta": 1, "emilio": 1, "karima": 1, "linnea": 1, "nikolina": 1, "oscar": 1 }, "perTask": { "powerpoint-grounding-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.320709228515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.320709228515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-powerpoint-deck passed 21 deterministic craftbook checks", "durationMs": 1950561, "timeoutMs": 4615385, "budgetUsedFraction": 0.4226215147815404 }, "timing": { "startedAt": "2026-11-05T18:07:11.971Z", "finishedAt": "2026-11-05T18:39:42.532Z", "timeToFirstArtifactMs": 301167, "timeToLastArtifactWriteMs": 1884203, "timeToFirstTokenMs": 37418, "firstTurnTtftMs": 23651, "timeToFirstToolCallMs": 45618 }, "team": { "totalGezelsCreated": 10, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Researcher", "Planner", "Copywriter", "Reviewer", "Designer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 34, "byTool": { "read_file": 6, "search": 1, "write_artifact": 6, "read_artifact": 6, "write_file": 1, "write_task_note": 2, "advance_task_step": 3, "validate": 1, "list_roots": 1, "convert_document": 1, "inspect_document": 2, "preview_document": 2, "save_artifact": 1, "copy_artifact_to_workspace": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [ { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-05_183515_new-session/tool-0-img-0.png", "bytes": 151081 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-05_183515_new-session/tool-0-img-1.png", "bytes": 140382 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-05_183515_new-session/tool-0-img-2.png", "bytes": 128164 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-05_183515_new-session/tool-0-img-3.png", "bytes": 138961 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-05_183515_new-session/tool-1-img-0.png", "bytes": 156318 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-05_183515_new-session/tool-1-img-1.png", "bytes": 152516 } ], "otherFileCount": 9 }, "sniff": { "progression": [ { "atMs": 8113, "filePath": "craftbook-powerpoint-deck", "bytes": 0, "score": 1, "scoreMax": 21, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/brief.md" }, { "atMs": 48996, "filePath": "craftbook-powerpoint-deck", "bytes": 0, "score": 2, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 305421, "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 4, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 668684, "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 6, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 774920, "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 12, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 1808563, "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 13, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 1832240, "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 15, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 1879557, "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 18, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 1885358, "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 20, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 1938568, "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-powerpoint-deck", "bytes": 2938, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T18:07:11.975Z] [trial] id=craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59 scenario=craftbook-powerpoint-deck model=underdog-saluki-27b-iq2-mix", "[2026-11-05T18:07:11.976Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-powerpoint-deck-NaAuwx", "[2026-11-05T18:07:11.976Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T18:07:11.976Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "[2026-11-05T18:07:11.976Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T18:07:11.978Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T18:07:11.983Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T18:07:11.989Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-powerpoint-deck-NaAuwx", "[2026-11-05T18:07:11.990Z] [trial] throughput-scaled ceiling: 35m → 77m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T18:07:17.942Z] [trial] daemon spawned pid=18196 port=https://127.0.0.1:55062", "[2026-11-05T18:07:19.257Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T18:07:19.277Z] [trial] meester=linnea", "[2026-11-05T18:07:19.277Z] [trial] running scenario setup hook", "[2026-11-05T18:07:19.870Z] [trial] scenario setup complete", "[2026-11-05T18:07:19.872Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T18:07:19.930Z] [poll] initial digests hard=853995a6eff95a3d soft=2bfd5897cd18f20b", "[2026-11-05T18:07:20.134Z] [poll] hard-progress: 853995a6eff95a3d → 1536a8b3c408186d after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T18:07:25.276Z] [poll] hard-progress: 1536a8b3c408186d → ac16fc90f41fb4ed after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T18:08:01.033Z] [poll] hard-progress: ac16fc90f41fb4ed → 076c74215cffdcb5 after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=35,stream=75) sniff=2/0B)", "[2026-11-05T18:08:30.482Z] [poll] hard-progress: 076c74215cffdcb5 → 3b84d6fdcf494c95 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=107,stream=293) sniff=2/0B)", "[2026-11-05T18:10:22.942Z] [poll] hard-progress: 3b84d6fdcf494c95 → 836f88b4bb327db3 after 112s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=403,stream=1882) sniff=2/0B)", "[2026-11-05T18:12:17.435Z] [poll] hard-progress: 836f88b4bb327db3 → 778ee8244935b5ff after 114s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=707,stream=3481) sniff=4/2938B)", "[2026-11-05T18:12:23.293Z] [poll] hard-progress: 778ee8244935b5ff → 9e90db440e2651cd after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=707,stream=3481) sniff=4/2938B)", "[2026-11-05T18:12:58.905Z] [poll] hard-progress: 9e90db440e2651cd → a8fd8abc1b8d5e98 after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=737,stream=3549) sniff=4/2938B)", "[2026-11-05T18:16:27.627Z] [poll] hard-progress: a8fd8abc1b8d5e98 → 0f75f7a2d698e87e after 209s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=1302,stream=6010) sniff=4/2938B)" ] }, "metrics": { "process": { "sampleCount": 326, "peakRssMb": 15087, "peakCpuPercent": 1312.8, "firstAtMs": 1, "lastAtMs": 1931254 }, "gpu": { "available": true, "sampleCount": 326, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 326, "peakUsedMb": 31446, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 83369, "totalOutputTokens": 13813, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 85004, "todayTokensCached": 81409, "todayTokensOut": 2061, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 85004, "totalTokensCached": 81409, "totalTokensOut": 2061, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 36916, "cachedInputTokens": 36847, "outputTokens": 48, "cost": 0, "durationMs": 242927, "outputTokensPerSec": 8.459664320519762, "at": "2026-11-05T18:39:27.351Z" }, "lastUpdated": "2026-11-05T18:39:27.351Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T18:39:31.016Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1932639, "meanTokensPerSec": 8.55, "genTokensPerSec": 8.55, "genTokens": 13813, "promptTokensPerSec": 370.98, "peakRssMb": 15087, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 85004, "output": 2061 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T18:07:11.971Z", "finishedAt": "2026-11-05T18:39:42.532Z", "timeToFirstArtifactMs": 301167, "timeToLastArtifactWriteMs": 1884203, "timeToFirstTokenMs": 37418, "firstTurnTtftMs": 23651, "timeToFirstToolCallMs": 45618 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "powerpoint-grounding-eval/1", "stepId": "research", "activatedAt": "2026-11-05T18:07:19.630Z", "completedAt": "2026-11-05T18:12:13.334Z", "ms": 293704, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "57077318-c602-4166-9e2d-252e282ebf1a" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "outline", "activatedAt": "2026-11-05T18:12:13.336Z", "completedAt": "2026-11-05T18:18:18.182Z", "ms": 364846, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "818d2c63-2b6d-4ea2-b11d-63f3878ced43" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "write", "activatedAt": "2026-11-05T18:18:18.183Z", "completedAt": "2026-11-05T18:20:25.536Z", "ms": 127353, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "d55597af-1774-4e26-b01c-2b58b4bf6707" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "review", "activatedAt": "2026-11-05T18:20:25.537Z", "completedAt": "2026-11-05T18:35:10.737Z", "ms": 885200, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "9f9c1cdc-93e9-4a0d-bc61-2f6600aa4e45" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "save", "activatedAt": "2026-11-05T18:35:10.739Z", "completedAt": "2026-11-05T18:39:27.287Z", "ms": 256548, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "839ab60d-a7ff-4823-9776-5c47c9e10852" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T18:07:17.977Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T18:07:17.869Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 293704 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "clemence": 1, "elzbieta": 1, "emilio": 1, "karima": 1, "linnea": 1, "nikolina": 1, "oscar": 1 }, "perTask": { "powerpoint-grounding-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.320709228515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.320709228515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "scenarioId": "craftbook-powerpoint-deck", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-05T19:14:25.517Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "artifactKind": "markdown", "artifactBytes": 1784, "artifactSha256": "3f20cd4621a889fc3f154209a1f6bb1e3cee8ed4f6e875f829c5e90e7d94e784", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/po/powerpoint-deck/versions/1.7.19/test.json", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ], "scoreAxes": { "source-acquisition": 8, "outline-parity": 9, "grounding": 6, "convertibility": 7 }, "meanScore": 7.5, "justification": "Exact source was read before outlining; all six titles match and every supplied metric/action survives. The source converts to a real six-slide PPTX. Slide 2 asserts an unsupported causal link and slide 5 repeats it as a headline despite a caveat. Dense citations and repeated numbers reduce presentation quality; tables become plain text in the saved deck.", "reviewBasis": "Read the complete 1,784-byte declared Markdown, 381-byte authoritative source, 2,950-byte source packet, 3,398-byte outline and 4,815-byte review. Audited all tool receipts, both saved PPTX hashes and slide XML. Viewed all six recorded PNG previews: these explicitly use reconstructed-import and do not establish native PowerPoint pixel fidelity. Source/outline parity and actual content are assessed separately from preview typography.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "status": "scored", "bytes": 1784, "sha256": "3f20cd4621a889fc3f154209a1f6bb1e3cee8ed4f6e875f829c5e90e7d94e784" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/workspace/powerpoint-grounding-eval/source/brief.md", "status": "reviewedSupportingArtifact", "bytes": 381, "sha256": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "status": "reviewedSupportingArtifact", "bytes": 2950, "sha256": "dd8668daaf43cd2a6627bfde6c933aba98460aba3e76a259f142cd73544856ac" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "status": "reviewedSupportingArtifact", "bytes": 3398, "sha256": "b3429eb36add706d9d49856e586eb4a786035aeba65a4ec5599f7720f0327afa" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/review.md", "status": "reviewedSupportingArtifact", "bytes": 4815, "sha256": "3ade48915d3cededc76a5b50cf9628dbff8f8b79d81ced07c1fd615fd7e6b4ad" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/deck.pptx", "status": "reviewedSupportingArtifact", "bytes": 35904, "sha256": "aa471476e9db16be7694e55c6e7131c71c56781ff457fffc4130a50e3c285661" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/workspace/powerpoint-grounding-eval/deliverables/d-day.pptx", "status": "reviewedSupportingArtifact", "bytes": 35904, "sha256": "aa471476e9db16be7694e55c6e7131c71c56781ff457fffc4130a50e3c285661" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:44.900Z", "reviewId": "062", "cellId": "r2-saluki-cuda-productivity-craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "scenarioId": "craftbook-powerpoint-deck", "status": "scored", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-powerpoint-deck", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "bfce7a39e1e7829294fc5e6da51ff615daa80e909f1fecc8c509cf169461f994", "evidenceSha256": "a1ba96b29484521e704772c5d71f7900d2ce8d65e2aff66a726b9d277248bdfa", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "bytes": 1784, "sha256": "3f20cd4621a889fc3f154209a1f6bb1e3cee8ed4f6e875f829c5e90e7d94e784", "shownBytes": 1784, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/workspace/powerpoint-grounding-eval/source/brief.md", "bytes": 381, "sha256": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "shownBytes": 381, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "bytes": 2950, "sha256": "dd8668daaf43cd2a6627bfde6c933aba98460aba3e76a259f142cd73544856ac", "shownBytes": 2950, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "bytes": 3398, "sha256": "b3429eb36add706d9d49856e586eb4a786035aeba65a4ec5599f7720f0327afa", "shownBytes": 3398, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/review.md", "bytes": 4815, "sha256": "3ade48915d3cededc76a5b50cf9628dbff8f8b79d81ced07c1fd615fd7e6b4ad", "shownBytes": 4815, "truncated": false } ], "contextFiles": [], "scoreAxes": { "source-acquisition": 8, "outline-parity": 9, "grounding": 7, "convertibility": 7 }, "justification": "sources.md records the brief as sole authority with a fact ledger and boundaries. Deck has 6 H1s matching the outline titles in order. All metrics trace to the brief. Weaknesses: \"silence drives preventable refunds\" is unsourced, and \"Headline stat: coverage of 18 SKUs\" is fluff. Slides are concise.", "limitations": "No rendering or PPTX conversion evidence. The source-reading step is inferred from sources.md, not a trace. The output path differs from the original prompt's, and the outline cites deliverables/d-day.pptx. Convertibility judged from the Markdown only.", "meanScore": 7.75, "durationMs": 14913, "costUsd": 0.0375112, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 8463, "cache_read_input_tokens": 852, "output_tokens": 357, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 8463, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 357, "cache_read_input_tokens": 852, "cache_creation_input_tokens": 8463, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 8463 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 357, "cacheReadInputTokens": 852, "cacheCreationInputTokens": 8463, "webSearchRequests": 0, "costUSD": 0.0375112, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 8463, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-powerpoint-deck / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 32m 31s; budget used: 42.3%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 301167`; `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":2938,\"score\":21,\"scoreMax\":21,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.4226215147815404`; `facts.toolUse.totalToolCalls: 34`; `facts.timing.timeToFirstArtifactMs: 301167`; `facts.outcome.durationMs: 1950561` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15087 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31446 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.55 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 83369 + 13813 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 294s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.71 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 32% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 301167`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":2938,\"score\":21,\"scoreMax\":21,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.4226215147815404`\n- `facts.toolUse.totalToolCalls: 34`\n- `facts.timing.timeToFirstArtifactMs: 301167`\n- `facts.outcome.durationMs: 1950561`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 301167`; `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":2938,\"score\":21,\"scoreMax\":21,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.4226215147815404`; `facts.toolUse.totalToolCalls: 34`; `facts.timing.timeToFirstArtifactMs: 301167`; `facts.outcome.durationMs: 1950561`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 8.5/10** is unchanged. Declared advisory axes: source-acquisition 8/10; outline-parity 9/10; grounding 6/10; convertibility 7/10. Mean **7.50/10**, codex-inline/current-session.\n\nExact source was read before outlining; all six titles match and every supplied metric/action survives. The source converts to a real six-slide PPTX. Slide 2 asserts an unsupported causal link and slide 5 repeats it as a headline despite a caveat. Dense citations and repeated numbers reduce presentation quality; tables become plain text in the saved deck.\n\nRead the complete 1,784-byte declared Markdown, 381-byte authoritative source, 2,950-byte source packet, 3,398-byte outline and 4,815-byte review. Audited all tool receipts, both saved PPTX hashes and slide XML. Viewed all six recorded PNG previews: these explicitly use reconstructed-import and do not establish native PowerPoint pixel fidelity. Source/outline parity and actual content are assessed separately from preview typography.\n\n## Positives\n\n- All 21 fixed checks pass; five workflow stages complete with five gate approvals and no gate rejections (outcome.success; sniff.latest.score=21, scoreMax=21; continuity.steps). The research source was read at18:07:57Z before the outline and cited throughout (toolUse.byTool.read_file=6; recording/chat-events.jsonl).\n- Six numbered outline titles exactly match six Markdown H1s and six saved PPTX slide text titles. The 18-SKU coverage, 18→6 hours, 14.2→8.9% and all three actions are retained. Both saved binaries are35,904 bytes and SHA256 aa471476e9db16be7694e55c6e7131c71c56781ff457fffc4130a50e3c285661 (artifacts.otherFileCount=9; toolUse.byTool.convert_document/save_artifact/copy_artifact_to_workspace; advisory-pptx-inspection.json).\n- Actual DocBlocks2.7.5 conversion, six reconstructed slide previews, artifact saving, workspace copying and inspection all occurred. The observed docblocks:// artifact and matching hash establish real conversion rather than the mock constants described in the sidecar (toolUse.byTool; artifacts.imageFiles[0..5]; advisory-tool-evidence.json).\n\n## Negatives\n\n- Slide2 says status silence drives refunds without source support; slide5 promotes the same causal story in its title. Labeling one bullet a proposal does not establish the causal claim elsewhere, especially when status silence is still an unresolved complaint. The reviewer notices this but still marks all criteria PASS (sniff.latest full fixed pass; toolUse.byTool.write_artifact=6; deck.md and review.md).\n- The source has repeated figures/citations and meta copy such as “No invented owners or dates.” Saved slide XML has no native tables: both Markdown comparisons flatten to text. Reconstructed previews also merge their column text; generic Slide N preview headings are an import representation, not evidence that native title text was lost (artifacts.imageFiles; advisory-pptx-inspection.json). Native PowerPoint layout was not audited.\n- 34 calls reduce fixed efficiency to2.5; review alone takes885,200ms. Three write attempts were rejected: supplied task constraints and supported 18h→6h shorthand were flagged, and even validate-reported1784 bytes was not recognized as evidence. Rewrites preserve actual unsupported causal prose (toolUse.totalToolCalls=34; toolUse.byTool.write_artifact=6; continuity.steps.perStep; advisory-tool-evidence.json).\n\n## Performance and GPU fit\n\nElapsed 32.51 minutes; 13813 recorded generated tokens at **8.55 tok/s**, prefill 370.98 tok/s. Peak VRAM **10642/12227 MiB**, process-tree RSS **15087 MiB**, whole-system RAM **31446/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications; shared GPU memory and paging were not measured.\n\nNative0.1.57 CUDA, one launch, 65,536-token allocation, one slot, f16KV and reasoning budget4096. Dense FFNs for39/64 layers on GPU and25 on CPU. This is hybrid residency. No failed endpoint, explicit OOM or context overflow was found in the trial log. Final captured maximum session fill 32.1% is a session snapshot, not a full peak-KV measurement. No mid-turn compactions. Performance remains separate from fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: in the Gilde powerpoint-deck craftbook.json, steps[id=review].prompt, add exactly: “A proposal label applies only to its sentence. Check causal titles and repeated claims separately; keep an unresolved problem separate from a proven cause.” Use the1.7.19 source as the candidate baseline. Measure supported-claim precision and review duration against this trial’s full gate pass plus unsupported causal statements; do not add pilot names or figures to model prompts.\n\n## Strategic experiment — not applied\n\nProposed experiment, not applied: in packages/service/src/chat/evidence-ledger.ts and packages/core/src/grounding/factual-writing.ts, preserve typed evidence for user constraints and validation receipts, and accept quantity-preserving unit abbreviations through packages/core/src/checks/claims.ts. Keep citation resolution distinct from entailment. This targets the three rejected supported writes without relaxing the guard for unsupported historical or causal claims. Separately characterize table-preservation loss at the DocBlocks conversion boundary before changing the external Squisq implementation.\n\n## Generalization and next experiment\n\nR1 failed4.6 with Markdown but no saved binaries; R2 passes8.5 with both real files, at the same frozen native/tuning identity. This is repeat variability, not an upgrade effect. Cross-check the proposed source/claim handling against the unrelated wikipedia-research-brief trial, whose initial ungrounded write was usefully rejected, and docblocks-theme-roundtrip, whose fixed pass also concealed unsupported interpretation.\n\nEvaluate the proposed source/claim experiment separately across the indicated unrelated tasks; measure supported-claim precision, false write rejections, completion and elapsed time. No source, tuning, prompt, gate, model or campaign-state change was made, and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/facts.json): fail, fixed4.6, 73.22 minutes, 2.82tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/facts.json): pass, fixed8.5, 2.56 minutes, 125.67tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-06-52-078Z-dnoj/facts.json): pass, fixed8.5, 2.83 minutes, 121.38tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are contextual; no causal native-version improvement is inferred.\n\n## Strategic note\n\nWorkflow completion improved in this repetition, while interpretation remains a recurring advisory defect. Preview and inspection success prove file generation and structural readability; they do not certify factual causation or exact native presentation quality.\n\n## Raw evidence\n\nSee facts.json, score.json, historical-comparison.json, advisory-artifact-review.json, advisory-tool-evidence.json and daemon.log. Also see advisory-pptx-inspection.json and the six recorded preview PNGs.\n" }, { "cellId": "r2-saluki-cuda-productivity-wikipedia-research-brief", "baselineCellId": "r2-saluki-cuda-productivity-wikipedia-research-brief", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T18:42:31.351Z", "finishedAt": "2026-11-05T18:59:37.675Z", "durationMs": 1026324, "success": true, "reason": "research brief is tool-grounded and passes all content gates (word-band, ordered-sections, menabrea-1842, lovelace-1843, notes-a-g, bernoulli, babbage-design, not-completed, source-citations, no-decoy-citation, claim-calibration, no-false-completion, mcp-research-provenance)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 845325 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "wikipedia-research", "bytes": 6295, "score": 13, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "claim-calibration", "no-false-completion", "mcp-research-provenance" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.22237018146915155 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 845325 }, { "source": "facts", "path": "outcome.durationMs", "value": 1026324 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 171, "peakRssMb": 13362, "peakCpuPercent": 1218.6, "firstAtMs": 0, "lastAtMs": 1009918 }, "gpu": { "available": true, "sampleCount": 171, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 171, "peakUsedMb": 30853, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26294, "totalOutputTokens": 8084, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 45198, "todayTokensCached": 30659, "todayTokensOut": 944, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 45198, "totalTokensCached": 30659, "totalTokensOut": 944, "totalCost": 0, "medianOutputTokensPerSec": 9.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.2, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13551, "cachedInputTokens": 11846, "outputTokens": 99, "cost": 0, "durationMs": 22521, "outputTokensPerSec": 9.206387767604093, "at": "2026-11-05T18:59:27.455Z" }, "lastUpdated": "2026-11-05T18:59:27.455Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T18:59:28.958Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1011258, "meanTokensPerSec": 8.84, "genTokensPerSec": 8.84, "genTokens": 8084, "promptTokensPerSec": 406.18, "peakRssMb": 13362, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 45198, "output": 944 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T18:42:37.207Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T18:42:37.138Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "elian": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 8, "maxContextFill": 0.166778564453125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.166778564453125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "research brief is tool-grounded and passes all content gates (word-band, ordered-sections, menabrea-1842, lovelace-1843, notes-a-g, bernoulli, babbage-design, not-completed, source-citations, no-decoy-citation, claim-calibration, no-false-completion, mcp-research-provenance)", "durationMs": 1026324, "timeoutMs": 4615385, "budgetUsedFraction": 0.22237018146915155 }, "timing": { "startedAt": "2026-11-05T18:42:31.351Z", "finishedAt": "2026-11-05T18:59:37.675Z", "timeToFirstArtifactMs": 845325, "timeToLastArtifactWriteMs": 1016087, "timeToFirstTokenMs": 26473, "firstTurnTtftMs": 14562, "timeToFirstToolCallMs": 188350 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "write_file": 2, "search_wikipedia": 1, "read_wikipedia_sources": 1, "read_file": 2, "replace_in_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 3 }, "sniff": { "progression": [ { "atMs": 845365, "filePath": "wikipedia-research", "bytes": 6230, "score": 11, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-false-completion", "mcp-research-provenance" ], "failReason": "WIKI-ENGINE-BAND is a synthpop band, not a source for this brief — remove the citation" }, { "atMs": 969967, "filePath": "wikipedia-research", "bytes": 6211, "score": 12, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "no-false-completion", "mcp-research-provenance" ], "failReason": "every mention of the “first programmer” label must be hedged in the same sentence as debated shorthand" }, { "atMs": 993453, "filePath": "wikipedia-research", "bytes": 6248, "score": 12, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "no-false-completion", "mcp-research-provenance" ], "failReason": "every mention of the “first programmer” label must be hedged in the same sentence as debated shorthand" }, { "atMs": 1017085, "filePath": "wikipedia-research", "bytes": 6295, "score": 13, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "claim-calibration", "no-false-completion", "mcp-research-provenance" ], "failReason": null } ], "latest": { "filePath": "wikipedia-research", "bytes": 6295, "score": 13, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "claim-calibration", "no-false-completion", "mcp-research-provenance" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T18:42:31.355Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "[2026-11-05T18:42:31.355Z] [trial] id=wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc scenario=wikipedia-research-brief model=underdog-saluki-27b-iq2-mix", "[2026-11-05T18:42:31.355Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T18:42:31.355Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-wikipedia-research-brief-EXZ31W", "[2026-11-05T18:42:31.355Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T18:42:31.358Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T18:42:31.361Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T18:42:31.366Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-wikipedia-research-brief-EXZ31W", "[2026-11-05T18:42:31.366Z] [trial] throughput-scaled ceiling: 35m → 77m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T18:42:37.135Z] [trial] daemon spawned pid=34080 port=https://127.0.0.1:51271", "[2026-11-05T18:42:38.553Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T18:42:38.568Z] [trial] meester=ines", "[2026-11-05T18:42:38.568Z] [trial] running scenario setup hook", "[2026-11-05T18:42:38.971Z] [trial] scenario setup complete", "[2026-11-05T18:42:38.973Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T18:42:39.043Z] [poll] initial digests hard=f6a4bedbc4076017 soft=097ca0801aa1e8c3", "[2026-11-05T18:42:39.125Z] [poll] hard-progress: f6a4bedbc4076017 → b08a6cb2e0e303b9 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T18:45:41.182Z] [poll] hard-progress: b08a6cb2e0e303b9 → a062b9fa79c9a3f1 after 182s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=449,stream=2616) sniff=0/0B)", "[2026-11-05T18:45:58.874Z] [poll] hard-progress: a062b9fa79c9a3f1 → ea382efc7fc3d1fc after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=461,stream=2636) sniff=0/0B)", "[2026-11-05T18:46:04.863Z] [poll] hard-progress: ea382efc7fc3d1fc → bebe731c3e463a3d after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=470,stream=2658) sniff=0/0B)", "[2026-11-05T18:56:36.860Z] [poll] hard-progress: bebe731c3e463a3d → a527b2a5a0898aba after 632s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=2333,stream=9678) sniff=11/6230B)", "[2026-11-05T18:56:48.627Z] [poll] hard-progress: a527b2a5a0898aba → 94b8ecdc2629668f after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=2354,stream=9739) sniff=11/6230B)", "[2026-11-05T18:58:41.410Z] [poll] hard-progress: 94b8ecdc2629668f → 473b093215e9c5ca after 113s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=6,slot=2585,stream=10468) sniff=12/6211B)", "[2026-11-05T18:59:04.919Z] [poll] hard-progress: 473b093215e9c5ca → 9e79ec426352a338 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=7,slot=2632,stream=10704) sniff=12/6248B)", "[2026-11-05T18:59:16.764Z] [poll] hard-progress: 9e79ec426352a338 → be272d286f2b0b20 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=8,slot=2642,stream=10723) sniff=12/6248B)" ] }, "metrics": { "process": { "sampleCount": 171, "peakRssMb": 13362, "peakCpuPercent": 1218.6, "firstAtMs": 0, "lastAtMs": 1009918 }, "gpu": { "available": true, "sampleCount": 171, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 171, "peakUsedMb": 30853, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26294, "totalOutputTokens": 8084, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 45198, "todayTokensCached": 30659, "todayTokensOut": 944, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 45198, "totalTokensCached": 30659, "totalTokensOut": 944, "totalCost": 0, "medianOutputTokensPerSec": 9.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.2, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13551, "cachedInputTokens": 11846, "outputTokens": 99, "cost": 0, "durationMs": 22521, "outputTokensPerSec": 9.206387767604093, "at": "2026-11-05T18:59:27.455Z" }, "lastUpdated": "2026-11-05T18:59:27.455Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T18:59:28.958Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1011258, "meanTokensPerSec": 8.84, "genTokensPerSec": 8.84, "genTokens": 8084, "promptTokensPerSec": 406.18, "peakRssMb": 13362, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 45198, "output": 944 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T18:42:31.351Z", "finishedAt": "2026-11-05T18:59:37.675Z", "timeToFirstArtifactMs": 845325, "timeToLastArtifactWriteMs": 1016087, "timeToFirstTokenMs": 26473, "firstTurnTtftMs": 14562, "timeToFirstToolCallMs": 188350 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T18:42:37.207Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T18:42:37.138Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "elian": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 8, "maxContextFill": 0.166778564453125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.166778564453125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-wikipedia-research-brief", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "scenarioId": "wikipedia-research-brief", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-05T19:14:25.639Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/workspace/analytical-engine-research-brief/research-brief.md", "artifactKind": "markdown", "artifactBytes": 6327, "artifactSha256": "69f24b180c7892fa3d845fe45f8d2161070f05f3ae09a321204b1d0bc7b270e0", "axisSource": "evals/src/scenarios/wikipedia-research-brief.ts", "axes": [ { "name": "synthesis", "description": "Builds an argument across sources rather than listing source summaries." }, { "name": "grounding", "description": "Claims and dates stay faithful to the signed local source cards." }, { "name": "citationQuality", "description": "Source ids appear near the claims they support and make provenance easy to audit." }, { "name": "calibration", "description": "Separates documented facts, interpretation, and disputed historical shorthand." } ], "scoreAxes": { "synthesis": 5, "grounding": 4, "citationQuality": 6, "calibration": 5 }, "meanScore": 5, "justification": "The brief preserves the central dates, design facts and disputed-label caveat, with auditable source IDs. It still claims an unsupported earliest account, invents details of the authorship debate, and misattributes a snapshot statement to Lovelace’s own account. The timeline is out of order and the long middle largely repeats individual source cards.", "reviewBasis": "Read all6,327 bytes of the final brief and the complete four returned local source cards plus search/read receipts. The scenario is hermetic: evaluation uses its supplied evidence only. No live web enrichment was used. Central chronology and citations pass fixed gates, but semantic support was assessed independently sentence by sentence.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/workspace/analytical-engine-research-brief/research-brief.md", "status": "scored", "bytes": 6327, "sha256": "69f24b180c7892fa3d845fe45f8d2161070f05f3ae09a321204b1d0bc7b270e0" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:01:48.496Z", "reviewId": "063", "cellId": "r2-saluki-cuda-productivity-wikipedia-research-brief", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "scenarioId": "wikipedia-research-brief", "status": "scored", "axes": [ { "name": "synthesis", "description": "Builds an argument across sources rather than listing source summaries." }, { "name": "grounding", "description": "Claims and dates stay faithful to the signed local source cards." }, { "name": "citationQuality", "description": "Source ids appear near the claims they support and make provenance easy to audit." }, { "name": "calibration", "description": "Separates documented facts, interpretation, and disputed historical shorthand." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "baselineCellId": "r2-saluki-cuda-productivity-wikipedia-research-brief", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "59dcb2c869014ebdbac3d502b60833fe0440d90e8612bff9194aa6758c9f9971", "evidenceSha256": "814ee012410acf7e2b6b0a4b25608658ffbae2dcc9a66eb03bcd7b8662a4de38", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/workspace/analytical-engine-research-brief/research-brief.md", "bytes": 6327, "sha256": "69f24b180c7892fa3d845fe45f8d2161070f05f3ae09a321204b1d0bc7b270e0", "shownBytes": 6327, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc\\workspace\\analytical-engine-research-brief\\mocks\\services.md", "bytes": 886, "sha256": "7809b583d708299ab83a2a997331fc3f0718e96d399f0c41a4dba1c6f91315b2", "shownBytes": 886, "truncated": false } ], "scoreAxes": { "synthesis": 5, "grounding": 5, "citationQuality": 7, "calibration": 7 }, "justification": "Structure, word count, and required sections are met. Citations sit inline and the off-topic band hit is excluded. \"First programmer\" is consistently hedged. However, the brief adds unsupported claims: \"earliest published account\", \"Lovelace's own account confirms\", and the debate's specifics. The timeline is out of order. Synthesis is mostly per-source summary.", "limitations": "The source cards and tool-call transcript were not shown, so grounding is judged from internal consistency and apparent overreach. Word count is estimated from byte size at roughly 950 words. Whether the Notes-versus-paper length claim matches the Menabrea card could not be verified.", "meanScore": 6, "durationMs": 17286, "costUsd": 0.029886, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5528, "cache_read_input_tokens": 0, "output_tokens": 777, "output_tokens_details": { "thinking_tokens": 419 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5528, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 777, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 5528, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5528 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 777, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 5528, "webSearchRequests": 0, "costUSD": 0.029886, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 419, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5528, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - wikipedia-research-brief / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 17m 6s; budget used: 22.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 845325`; `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":6295,\"score\":13,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.22237018146915155`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 845325`; `facts.outcome.durationMs: 1026324` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13362 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10632 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30853 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.84 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 26294 + 8084 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 17% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 845325`\n- `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":6295,\"score\":13,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.22237018146915155`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 845325`\n- `facts.outcome.durationMs: 1026324`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 845325`; `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":6295,\"score\":13,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.22237018146915155`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 845325`; `facts.outcome.durationMs: 1026324`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass, 10/10** is unchanged. Declared advisory axes: synthesis 5/10; grounding 4/10; citationQuality 6/10; calibration 5/10. Mean **5.00/10**, codex-inline/current-session.\n\nThe brief preserves the central dates, design facts and disputed-label caveat, with auditable source IDs. It still claims an unsupported earliest account, invents details of the authorship debate, and misattributes a snapshot statement to Lovelace’s own account. The timeline is out of order and the long middle largely repeats individual source cards.\n\nRead all6,327 bytes of the final brief and the complete four returned local source cards plus search/read receipts. The scenario is hermetic: evaluation uses its supplied evidence only. No live web enrichment was used. Central chronology and citations pass fixed gates, but semantic support was assessed independently sentence by sentence.\n\n## Positives\n\n- All13 fixed gates pass, including the700–1,500-word band, required section order,1842/1843 chronology, Notes A–G, Bernoulli, design/noncompletion facts and local MCP provenance. Search/read both succeeded and the final brief cites only the three relevant source IDs (outcome.success; sniff.latest.signals; toolUse.byTool.search_wikipedia=1 and read_wikipedia_sources=1).\n- The initial unsupported write was blocked, then tool retrieval and three small repairs produced the final brief. The disputed “first programmer” label is explicitly qualified and the band is excluded from the Sources section (toolUse.byTool.write_file=2; replace_in_file=3; advisory-tool-evidence.json). No missing roles, red flags or auto-answer interventions (team.missingExpectedRoles; toolUse.redFlags; autoAnswer.total).\n\n## Negatives\n\n- The opening “earliest published account” assertion is absent from the cards and conflicts with the brief’s own description of the earlier1842 account. The detailed modern-programming/predating/collaboration debate is also absent. “Lovelace’s own account confirms” incorrectly promotes a Wikipedia snapshot into primary testimony (sniff.latest full fixed pass; toolUse.byTool.read_wikipedia_sources=1; final brief versus returned source extracts).\n- The timeline places Pre-1842 after1843. The middle repeats the card contents under separate headings rather than sustaining a cross-source argument; source-limitation caveats recur at length. Some interpretations appear as factual claims despite nearby citations (artifacts.otherFileCount=3; sniff.latest ordered-sections and claim-calibration; research-brief.md).\n- The first attempted draft came before either research call and included unsupported dates/publication details. The guard prevented that artifact, but after evidence was present it accepted weaker unsupported superlatives and causal/interpretive elaboration (toolUse.byTool.write_file=2; recording/chat-events.jsonl at18:45:39Z,18:45:58Z,18:46:04Z and18:56:36Z). This is a recovered process error, not an engine fault.\n\n## Performance and GPU fit\n\nElapsed 17.11 minutes; 8084 recorded generated tokens at **8.84 tok/s**, prefill 406.18 tok/s. Peak VRAM **10632/12227 MiB**, process-tree RSS **13362 MiB**, whole-system RAM **30853/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications; shared GPU memory and paging were not measured.\n\nNative0.1.57 CUDA, one launch, 65,536-token allocation, one slot, f16KV and reasoning budget4096. Dense FFNs for39/64 layers on GPU and25 on CPU. This is hybrid residency. No failed endpoint, explicit OOM or context overflow was found in the trial log. Final captured maximum session fill 16.7% is a session snapshot, not a full peak-KV measurement. No mid-turn compactions. Performance remains separate from fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: append to factualWritingGuidance in packages/core/src/grounding/factual-writing.ts: “A source mentioning the topic does not support every claim about it. Verify superlatives, causes and explanations of a dispute; label an inference as an inference. A secondary summary is not testimony from the person it describes.” Measure unsupported-claim count on this brief and the unrelated PowerPoint/docblocks reports; preserve the current word bands and all gates.\n\n## Strategic experiment — not applied\n\nProposed experiment, not applied: extend packages/service/src/chat/evidence-ledger.ts with per-claim evidence spans and source-kind metadata, using packages/core/src/checks/claims.ts to distinguish cited entity presence from support for the full assertion. Surface unsupported superlatives or source-kind upgrades as a focused repair request. Keep the useful rejection of the initial evidence-free draft and test false positives against the presentation’s supported quantity abbreviations and task constraints.\n\n## Generalization and next experiment\n\nR1 failed3 with no brief; R2 passes10 with a substantive document after real local research. Both share the same frozen configuration, so this does not establish a native-version gain. The unrelated PowerPoint and DocBlocks repetitions likewise retain unsupported explanations under passing fixed gates; this makes source/interpretation separation a broad experiment rather than an eval-specific wording patch.\n\nEvaluate the proposed source/claim experiment separately across the indicated unrelated tasks; measure supported-claim precision, false write rejections, completion and elapsed time. No source, tuning, prompt, gate, model or campaign-state change was made, and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/facts.json): fail, fixed3, 44.39 minutes, 2.9tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/facts.json): pass, fixed9.5, 3.11 minutes, 106.88tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-16-14-281Z-fpgc/facts.json): pass, fixed9.5, 1.73 minutes, 111.98tok/s.\n\nOnly same-campaign repetitions share the frozen identity. Older runs are contextual; no causal native-version improvement is inferred.\n\n## Strategic note\n\nReliable retrieval and fixed completion are demonstrated in this repetition. Reliable synthesis is less settled: source names and uncertainty words satisfy the gate while unsupported prose survives. Advisory5.0 remains separate from fixed10.0.\n\n## Raw evidence\n\nSee facts.json, score.json, historical-comparison.json, advisory-artifact-review.json, advisory-tool-evidence.json and daemon.log.\n" }, { "cellId": "r2-saluki-cuda-productivity-craftbook-week-plan", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-week-plan", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T19:02:34.586Z", "finishedAt": "2026-11-05T19:34:48.603Z", "durationMs": 1934017, "success": true, "reason": "craftbook-week-plan passed 12 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 506905 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-week-plan", "bytes": 2864, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.3259176760083221 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 506905 }, { "source": "facts", "path": "outcome.durationMs", "value": 1934017 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 323, "peakRssMb": 14433, "peakCpuPercent": 1216.3, "firstAtMs": 0, "lastAtMs": 1921243 }, "gpu": { "available": true, "sampleCount": 323, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 323, "peakUsedMb": 31128, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 88790, "totalOutputTokens": 12444, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 72156, "todayTokensCached": 58119, "todayTokensOut": 10089, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 72156, "totalTokensCached": 58119, "totalTokensOut": 10089, "totalCost": 0, "medianOutputTokensPerSec": 8.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.2, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7108, "cachedInputTokens": 6871, "outputTokens": 178, "cost": 0, "durationMs": 71933, "outputTokensPerSec": 8.62752334951457, "at": "2026-11-05T19:33:36.293Z" }, "lastUpdated": "2026-11-05T19:33:36.293Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T19:34:42.812Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1922716, "meanTokensPerSec": 8.14, "genTokensPerSec": 8.14, "genTokens": 12444, "promptTokensPerSec": 437.03, "peakRssMb": 14433, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 72156, "output": 10089 } } } }, "craftbook": { "craftbookId": "week-plan", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "calendar-desk-week-plan-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T19:02:41.505Z", "completedAt": "2026-11-05T19:11:01.593Z", "ms": 500088, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "e3f7c24d-6d7a-45c6-9c69-983248f95805" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "plan", "activatedAt": "2026-11-05T19:11:01.595Z", "completedAt": "2026-11-05T19:22:08.455Z", "ms": 666860, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "e3f7c24d-6d7a-45c6-9c69-983248f95805" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "report", "activatedAt": "2026-11-05T19:22:08.456Z", "completedAt": "2026-11-05T19:25:03.930Z", "ms": 175474, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "621fb651-299b-4be1-b88f-9c7d478e4a9d" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T19:25:03.931Z", "completedAt": "2026-11-05T19:31:21.043Z", "ms": 377112, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "0325e7cf-fb19-422f-877b-f859f3c2fad2" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T19:31:21.044Z", "completedAt": "2026-11-05T19:33:36.251Z", "ms": 135207, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "c892236f-be7d-4e87-a948-c3d16e2ac042" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T19:02:40.098Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T19:02:40.004Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 377112 }, "sessions": { "total": 7, "taskScoped": 4, "byGezel": { "ayano": 1, "dagny": 1, "epeli": 1, "meike": 2, "moussa": 1, "ruslan": 1 }, "perTask": { "calendar-desk-week-plan-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.221893310546875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.221893310546875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/week-plan", "knowledge://bendyline/handboek/craftbook/weekly-menu-plan", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/plan", "knowledge://bendyline/handboek/craftbook/onboarding-checklist" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-week-plan passed 12 deterministic craftbook checks", "durationMs": 1934017, "timeoutMs": 5934066, "budgetUsedFraction": 0.3259176760083221 }, "timing": { "startedAt": "2026-11-05T19:02:34.586Z", "finishedAt": "2026-11-05T19:34:48.603Z", "timeToFirstArtifactMs": 506905, "timeToLastArtifactWriteMs": 1923957, "timeToFirstTokenMs": 42923, "firstTurnTtftMs": 29731, "timeToFirstToolCallMs": 57213 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 19, "byTool": { "list_artifacts": 1, "read_artifact": 7, "read_files": 2, "read_file": 2, "write_artifact": 4, "write_task_note": 1, "advance_task_step": 1, "copy_artifact_to_workspace": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 16 }, "sniff": { "progression": [ { "atMs": 7107, "filePath": "craftbook-week-plan", "bytes": 0, "score": 0, "scoreMax": 12, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: followups.json" }, { "atMs": 124509, "filePath": "craftbook-week-plan", "bytes": 0, "score": 1, "scoreMax": 12, "signals": [], "failReason": "task sourced from craftbook week-plan has not finished: its gates have not accepted the result yet" }, { "atMs": 1862500, "filePath": "craftbook-week-plan", "bytes": 0, "score": 2, "scoreMax": 12, "signals": [], "failReason": "tasks/eval/report.md is 0 bytes, need ≥ 1200" }, { "atMs": 1927755, "filePath": "craftbook-week-plan", "bytes": 2864, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-week-plan", "bytes": 2864, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T19:02:34.589Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "[2026-11-05T19:02:34.589Z] [trial] id=craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1 scenario=craftbook-week-plan model=underdog-saluki-27b-iq2-mix", "[2026-11-05T19:02:34.590Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T19:02:34.589Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-week-plan-FiALuu", "[2026-11-05T19:02:34.590Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T19:02:34.592Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T19:02:34.596Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T19:02:34.601Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-week-plan-FiALuu", "[2026-11-05T19:02:34.601Z] [trial] throughput-scaled ceiling: 45m → 99m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T19:02:40.098Z] [trial] daemon spawned pid=40564 port=https://127.0.0.1:59440", "[2026-11-05T19:02:41.004Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T19:02:41.020Z] [trial] meester=dagny", "[2026-11-05T19:02:41.020Z] [trial] running scenario setup hook", "[2026-11-05T19:02:41.605Z] [trial] scenario setup complete", "[2026-11-05T19:02:41.607Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T19:02:41.654Z] [poll] initial digests hard=d40a7c7942ca4646 soft=4bd080e0daeca924", "[2026-11-05T19:02:41.731Z] [poll] hard-progress: d40a7c7942ca4646 → 0a2f3e7b05bab311 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T19:02:46.861Z] [poll] hard-progress: 0a2f3e7b05bab311 → 5be1235f68052000 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T19:03:34.355Z] [poll] hard-progress: 5be1235f68052000 → c911b89b76e85791 after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=46,stream=97) sniff=0/0B)", "[2026-11-05T19:04:39.143Z] [poll] hard-progress: c911b89b76e85791 → 5dfa39a45e1fa23c after 65s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=213,stream=1027) sniff=1/0B)", "[2026-11-05T19:11:05.615Z] [poll] hard-progress: 5dfa39a45e1fa23c → dfd03b004034a5d3 after 386s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=1210,stream=5076) sniff=1/0B)", "[2026-11-05T19:14:09.862Z] [poll] hard-progress: dfd03b004034a5d3 → de573b79db9bf148 after 184s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=7,slot=1557,stream=6426) sniff=1/0B)", "[2026-11-05T19:22:09.052Z] [poll] hard-progress: de573b79db9bf148 → 7ebadc22fc03c8f1 after 479s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=8,slot=2775,stream=11252) sniff=1/0B)", "[2026-11-05T19:22:14.846Z] [poll] hard-progress: 7ebadc22fc03c8f1 → f089d5973179c5d9 after 6s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=4,tools=8,slot=2775,stream=11252) sniff=1/0B)", "[2026-11-05T19:22:44.546Z] [poll] hard-progress: f089d5973179c5d9 → 78fd47c3694f6ef3 after 30s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=4,tools=10,slot=2796,stream=11294) sniff=1/0B)" ] }, "metrics": { "process": { "sampleCount": 323, "peakRssMb": 14433, "peakCpuPercent": 1216.3, "firstAtMs": 0, "lastAtMs": 1921243 }, "gpu": { "available": true, "sampleCount": 323, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 323, "peakUsedMb": 31128, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 88790, "totalOutputTokens": 12444, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 72156, "todayTokensCached": 58119, "todayTokensOut": 10089, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 72156, "totalTokensCached": 58119, "totalTokensOut": 10089, "totalCost": 0, "medianOutputTokensPerSec": 8.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.2, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7108, "cachedInputTokens": 6871, "outputTokens": 178, "cost": 0, "durationMs": 71933, "outputTokensPerSec": 8.62752334951457, "at": "2026-11-05T19:33:36.293Z" }, "lastUpdated": "2026-11-05T19:33:36.293Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T19:34:42.812Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1922716, "meanTokensPerSec": 8.14, "genTokensPerSec": 8.14, "genTokens": 12444, "promptTokensPerSec": 437.03, "peakRssMb": 14433, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 72156, "output": 10089 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T19:02:34.586Z", "finishedAt": "2026-11-05T19:34:48.603Z", "timeToFirstArtifactMs": 506905, "timeToLastArtifactWriteMs": 1923957, "timeToFirstTokenMs": 42923, "firstTurnTtftMs": 29731, "timeToFirstToolCallMs": 57213 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "calendar-desk-week-plan-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T19:02:41.505Z", "completedAt": "2026-11-05T19:11:01.593Z", "ms": 500088, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "e3f7c24d-6d7a-45c6-9c69-983248f95805" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "plan", "activatedAt": "2026-11-05T19:11:01.595Z", "completedAt": "2026-11-05T19:22:08.455Z", "ms": 666860, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "e3f7c24d-6d7a-45c6-9c69-983248f95805" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "report", "activatedAt": "2026-11-05T19:22:08.456Z", "completedAt": "2026-11-05T19:25:03.930Z", "ms": 175474, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "621fb651-299b-4be1-b88f-9c7d478e4a9d" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T19:25:03.931Z", "completedAt": "2026-11-05T19:31:21.043Z", "ms": 377112, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "0325e7cf-fb19-422f-877b-f859f3c2fad2" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T19:31:21.044Z", "completedAt": "2026-11-05T19:33:36.251Z", "ms": 135207, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "c892236f-be7d-4e87-a948-c3d16e2ac042" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T19:02:40.098Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T19:02:40.004Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 377112 }, "sessions": { "total": 7, "taskScoped": 4, "byGezel": { "ayano": 1, "dagny": 1, "epeli": 1, "meike": 2, "moussa": 1, "ruslan": 1 }, "perTask": { "calendar-desk-week-plan-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.221893310546875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.221893310546875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-craftbook-week-plan", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "scenarioId": "craftbook-week-plan", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-05T20:07:26.871Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "artifactKind": "markdown", "artifactBytes": 2954, "artifactSha256": "bcc737b0c7a3abec6f21d575b1dbf19172e829593f91e069f7a96f4931d13af7", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/we/week-plan/versions/1.0.5/test.json", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ], "scoreAxes": { "coverage": 6, "prioritization": 4, "conflict-handling": 5, "realism": 4 }, "meanScore": 4.75, "justification": "All eight meetings and three follow-ups appear, with prep flags and a concrete overlap fix. Priya’s availability is invented and the proposed move reads as completed. Focus blocks fit, but the biggest work gets the smaller block, its SOW is pushed before09:00, and advertised free hours ignore the blocks. Working-hour assumptions disappear from the report.", "reviewBasis": "Read complete final2,954-byte report and identical artifact copy; source calendar’s eight events, three follow-ups, complete scope, plan and verdict. Audited tool chronology and deterministic interval arithmetic. Final source data unchanged. Intermediate noArtifact replaced only after the runner captured the final workspace output.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "bytes": 2954, "sha256": "bcc737b0c7a3abec6f21d575b1dbf19172e829593f91e069f7a96f4931d13af7", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md", "bytes": 2954, "sha256": "bcc737b0c7a3abec6f21d575b1dbf19172e829593f91e069f7a96f4931d13af7", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/scope.md", "bytes": 3252, "sha256": "f2d446853ff73f964b30100b4623420d9f5dc01c40b7681403fd67000f51c16d", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/plan.md", "bytes": 3331, "sha256": "dbbfb68291423173f120007c968015be1fd80398891ab89078e3643078b960e2", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/verdict.md", "bytes": 1270, "sha256": "ed02e41042b99629e59f16ecc43214f877c7624a7b2437a938403599a6dd9c6c", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:04.795Z", "reviewId": "064", "cellId": "r2-saluki-cuda-productivity-craftbook-week-plan", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "scenarioId": "craftbook-week-plan", "status": "scored", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-week-plan", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "04cec19bf85d21f202797b2b6e462f423a48f8958b438a237ac31be03cfdb78b", "evidenceSha256": "4c07fe94b6039e843ff5d2551577b6418d1c23df28164f8b851557ae4615ffcc", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "bytes": 2954, "sha256": "bcc737b0c7a3abec6f21d575b1dbf19172e829593f91e069f7a96f4931d13af7", "shownBytes": 2954, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md", "bytes": 2954, "sha256": "bcc737b0c7a3abec6f21d575b1dbf19172e829593f91e069f7a96f4931d13af7", "shownBytes": 2954, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/scope.md", "bytes": 3252, "sha256": "f2d446853ff73f964b30100b4623420d9f5dc01c40b7681403fd67000f51c16d", "shownBytes": 3252, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/plan.md", "bytes": 3331, "sha256": "dbbfb68291423173f120007c968015be1fd80398891ab89078e3643078b960e2", "shownBytes": 3331, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/verdict.md", "bytes": 1270, "sha256": "ed02e41042b99629e59f16ecc43214f877c7624a7b2437a938403599a6dd9c6c", "shownBytes": 1270, "truncated": false } ], "contextFiles": [], "scoreAxes": { "coverage": 8, "prioritization": 6, "conflict-handling": 8, "realism": 6 }, "justification": "All 8 events and 3 follow-ups appear, with prep flags. The Tuesday overlap has a one-decision fix (1:1 to 11:00–11:30). But Wednesday's 3h block is called \"the biggest work\" while Thursday's 4h block is larger. Wednesday is labelled \"crunch\" with 7h free. Priya being \"in Monday's vendor sync\" is unsupported.", "limitations": "Source calendar and followups JSON were not shown, so event counts and the Priya claim are checked only against the agent's own scope.md. The verdict.md is self-graded. No rendering or runtime verification.", "meanScore": 7, "durationMs": 22354, "costUsd": 0.0482966, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 9552, "cache_read_input_tokens": 846, "output_tokens": 1000, "output_tokens_details": { "thinking_tokens": 651 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 9552, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 1000, "cache_read_input_tokens": 846, "cache_creation_input_tokens": 9552, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9552 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 1000, "cacheReadInputTokens": 846, "cacheCreationInputTokens": 9552, "webSearchRequests": 0, "costUSD": 0.0482966, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 651, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 9552, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-week-plan / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 32m 14s; budget used: 32.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 506905`; `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":2864,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.3259176760083221`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 506905`; `facts.outcome.durationMs: 1934017` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14433 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31128 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.14 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 88790 + 12444 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 377s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 22% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 506905`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":2864,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.3259176760083221`\n- `facts.toolUse.totalToolCalls: 19`\n- `facts.timing.timeToFirstArtifactMs: 506905`\n- `facts.outcome.durationMs: 1934017`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 506905`; `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":2864,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.3259176760083221`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 506905`; `facts.outcome.durationMs: 1934017`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass9.5/10** is unchanged. Exact declared axes: coverage6, prioritization4, conflict-handling5, realism4. Advisory mean **4.75/10**, codex-inline/current-session.\n\nAll eight meetings and three follow-ups appear, with prep flags and a concrete overlap fix. Priya’s availability is invented and the proposed move reads as completed. Focus blocks fit, but the biggest work gets the smaller block, its SOW is pushed before09:00, and advertised free hours ignore the blocks. Working-hour assumptions disappear from the report.\n\nRead complete final2,954-byte report and identical artifact copy; source calendar’s eight events, three follow-ups, complete scope, plan and verdict. Audited tool chronology and deterministic interval arithmetic. Final source data unchanged. Intermediate noArtifact replaced only after the runner captured the final workspace output.\n\n## Positives\n\n- All12 deterministic checks pass; all eight calendar meetings, the two agenda-less meetings and three follow-ups are present. The Tuesday30-minute overlap and11:00–11:30 alternative are concrete (outcome.success; sniff.latest.score=12, scoreMax=12; final report versus calendar fixtures).\n- Five workflow stages complete, with four gate approvals, no rejections, one session reuse and no resume failures. A standard copy_artifact_to_workspace call at19:34:38Z retains identical report bytes on both surfaces after the workflow finishes (continuity.steps; continuity.sessions; toolUse.byTool.copy_artifact_to_workspace=1; artifact SHA hashes).\n- Friday is correctly identified as the day with most meeting time. The Wednesday/Thursday focus intervals fit Jordan’s visible calendar, and follow-ups are placed on their due days (artifacts.otherFileCount=16; toolUse.byTool.read_files=2; advisory-calendar-audit.json).\n\n## Negatives\n\n- The report invents Priya’s Tuesday availability from her attendance at Monday’s vendor meeting. The day list says the1:1 is “moved” while the conflict section still asks for a decision; the source remains at10:30. No calendar mutation or attendee acceptance occurred (toolUse.byTool contains only read/write/copy/task tools; sniff.latest full fixed pass; report versus raw event record).\n- The biggest work is unnamed and assigned to Wednesday’s3h block while Thursday gets4h. The SOW must be sent before the09:00 focus block, outside the supporting plan’s assumed working day. The final report drops that09:00–17:00 assumption, and its7h free labels omit the3h/4h focus commitments (artifacts.otherFileCount=16; report/plan; advisory-calendar-audit.json).\n- Reviewer marked “every prep note scheduled on an earlier day” PASS while citing same-day prep, and accepted the capacity picture without subtracting focus/follow-up commitments. Supporting plan also claims a drafted list/Sunday kickoff email with no evidence. These survive a no-repair workflow (continuity.steps.gateRejections=0; toolUse.byTool.write_artifact=4; verdict.md and plan.md). The user did not require earlier-day prep: that mismatch belongs to craftbook review semantics, not a reason to change the fixed score.\n\n## Performance and GPU fit\n\nElapsed **32.23min**,19 calls,12,444 recorded generation tokens at **8.14tok/s**, prefill437.03tok/s. Peak GPU memory **10642/12227MiB**; process-tree RSS **14433MiB**; whole-system RAM **31128/31865MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications. Shared GPU memory and paging were not measured.\n\nOne native0.1.57 CUDA launch;65,536-token allocation, one slot, f16KV, reasoning4096. Dense FFNs for39/64 layers on GPU and25 on CPU: hybrid residency. No failed endpoint, explicit OOM or context overflow found. Maximum captured session fill22.2% is a session snapshot, not a full native KV peak. No mid-turn compactions. Performance is separate from both fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nIn the Gilde week-plan craftbook.json based on1.0.5, steps[id=report].prompt, add: “Carry forward working-hour assumptions. Mark proposed moves as pending; another attendee’s availability is unknown without their calendar. Show remaining capacity after meetings, focus, prep and follow-ups, and name the work assigned to each focus block.” This directly targets the final report’s capacity and proposal-state losses (sniff.latest full pass; report/plan). Review steps[id=evaluate].prompt separately: requiring earlier-day prep even for a Monday-morning request is not equivalent to the user’s request. Do not silently weaken or reinterpret the rule during a running comparison.\n\n## Strategic experiment — not applied\n\nPropose a reusable interval-capacity check in a new packages/core/src/checks/calendar-capacity.ts, with optional dispatch from packages/service/src/tasks/gate-eval.ts. Take working hours, confirmed intervals and proposed commitments as separate typed inputs; report overlap, before-work tasks, unassigned focus blocks and net free time. Keep other-attendee availability unknown unless explicitly sourced. This would make the arithmetic/state distinction inspectable before prose review, without embedding fixture names or expected times in a model profile.\n\n## Generalization and next experiment\n\nR1 fixed9.0/advisory4.75 versus R2 fixed9.5/advisory4.75: timing improves and Friday’s relative load is corrected, but proposal-to-fact drift and unnamed priorities persist. Both use the same frozen native/tuning identity, so no causal version benefit is inferred. Compare the proposed assumptions/state handling against unrelated meeting-followup (schedule a legal review became a review deadline) and plan-and-estimate (readiness/dependency gaps); measure unsupported-state claims, feasible capacity and fixed completion before recommending a shared change. No source, prompt, gate, tuning, campaign-state or model file was changed, and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/facts.json): pass, fixed9, 92.88min, 2.64tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/facts.json): pass, fixed8.5, 6.21min, 117.06tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-29-26-682Z-ldgk/facts.json): pass, fixed8.5, 8.62min, 126.33tok/s.\n\nHistorical other-model/native-version runs are contextual, not controlled comparisons.\n\n## Strategic note\n\nCalendar retrieval and workflow completion work here. A full gate pass still does not establish an executable personal schedule: the unsupported attendee claim and ambiguous proposed change remain practical defects.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-calendar-audit.json, advisory-artifact-review.json, advisory-tool-evidence.json, historical-comparison.json, the final report/plan/verdict and daemon.log.\n" }, { "cellId": "r2-saluki-cuda-productivity-craftbook-ab-test-readout", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-ab-test-readout", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T19:37:35.763Z", "finishedAt": "2026-11-05T21:32:06.102Z", "durationMs": 6870339, "success": true, "reason": "craftbook-ab-test-readout passed 19 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 145549 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-ab-test-readout", "bytes": 474, "score": 19, "scoreMax": 19, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 0, "ruleId": "E_BOTH_SEVERE", "summary": "The trial exhausted its budget while making at least thirty tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 55 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 145549 }, { "source": "facts", "path": "outcome.durationMs", "value": 6870339 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8, "band": "ship-ready" }, "facts": { "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 1151, "peakRssMb": 14850, "peakCpuPercent": 1737.1, "firstAtMs": 0, "lastAtMs": 6859644 }, "gpu": { "available": true, "sampleCount": 1151, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1151, "peakUsedMb": 31543, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 103039, "totalOutputTokens": 56023, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 100660, "todayTokensCached": 94145, "todayTokensOut": 9352, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 100660, "totalTokensCached": 94145, "totalTokensOut": 9352, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3516, "outputTokens": 97, "cost": 0, "durationMs": 43113, "outputTokensPerSec": 9.064258796674322, "at": "2026-11-05T21:31:59.780Z" }, "lastUpdated": "2026-11-05T21:31:59.780Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T21:32:02.938Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6861078, "meanTokensPerSec": 8.65, "genTokensPerSec": 8.65, "genTokens": 56023, "promptTokensPerSec": 376.65, "peakRssMb": 14850, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 100660, "output": 9352 } } } }, "craftbook": { "craftbookId": "ab-test-readout", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 6, "completed": 5, "gateApprovals": 6, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "a-b-test-readout-eval/1", "stepId": "hypothesis-scope", "activatedAt": "2026-11-05T19:37:43.277Z", "completedAt": "2026-11-05T19:40:01.425Z", "ms": 138148, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "3d3ae99c-2248-47f2-9d77-89772306ff21" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "analyze", "activatedAt": "2026-11-05T19:40:01.426Z", "completedAt": "2026-11-05T20:14:18.049Z", "ms": 2056623, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "3d3ae99c-2248-47f2-9d77-89772306ff21" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "write", "activatedAt": "2026-11-05T20:30:06.997Z", "completedAt": "2026-11-05T21:21:53.421Z", "ms": 3106424, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "30fb088b-11df-4710-a31a-1f08c8eb8ae1", "f33a1e05-fc5f-4433-9c42-84a1330bfc75" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T21:21:53.422Z", "completedAt": "2026-11-05T21:28:45.168Z", "ms": 411746, "gateRejections": 1, "toolCalls": 26, "sessionIds": [ "834ccdd6-cea1-49ce-b2f8-7df57b7e884b", "be719aab-cb8d-4937-bee2-3eedc4eaa384" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T21:28:45.169Z", "completedAt": "2026-11-05T21:31:14.083Z", "ms": 148914, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "c57420a6-fe7b-4504-a046-236bf224c520" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T19:37:41.574Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T19:37:41.475Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 411746 }, "sessions": { "total": 7, "taskScoped": 6, "byGezel": { "dolores": 2, "nhung": 2, "noor": 1, "tolga": 2 }, "perTask": { "a-b-test-readout-eval/1": 6 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.8571428571428571, "maxMessages": 4, "maxContextFill": 0.205413818359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.205413818359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/ab-test-readout", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/how-we-test-models", "knowledge://bendyline/handboek/craftbook/cohort-analysis", "knowledge://bendyline/handboek/whats-new/1.26273" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-ab-test-readout passed 19 deterministic craftbook checks", "durationMs": 6870339, "timeoutMs": 5934066, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-05T19:37:35.763Z", "finishedAt": "2026-11-05T21:32:06.102Z", "timeToFirstArtifactMs": 145549, "timeToLastArtifactWriteMs": 6863999, "timeToFirstTokenMs": 41308, "firstTurnTtftMs": 27604, "timeToFirstToolCallMs": 69662 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Data Analyst", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 55, "byTool": { "read_file": 22, "write_artifact": 8, "write_file": 2, "run_installed_script": 1, "list_scripts": 1, "write_task_note": 6, "advance_task_step": 3, "read_artifact": 6, "set_task_status": 1, "read_task_notes": 3, "ask_user_question": 1, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 7697, "filePath": "craftbook-ab-test-readout", "bytes": 0, "score": 0, "scoreMax": 19, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/experiment-brief.md, data/checkout-test.csv" }, { "atMs": 72114, "filePath": "craftbook-ab-test-readout", "bytes": 0, "score": 1, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" }, { "atMs": 2147489, "filePath": "craftbook-ab-test-readout", "bytes": 476, "score": 10, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" }, { "atMs": 2439052, "filePath": "craftbook-ab-test-readout", "bytes": 476, "score": 17, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" }, { "atMs": 6819258, "filePath": "craftbook-ab-test-readout", "bytes": 476, "score": 18, "scoreMax": 19, "signals": [], "failReason": "data/stats.json significant_at_95 should equal true but was false: the two-proportion z-test gives z~2.66, p~0.008 — significant at 95%" }, { "atMs": 6866670, "filePath": "craftbook-ab-test-readout", "bytes": 474, "score": 19, "scoreMax": 19, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-ab-test-readout", "bytes": 474, "score": 19, "scoreMax": 19, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T19:37:35.767Z] [trial] id=craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18 scenario=craftbook-ab-test-readout model=underdog-saluki-27b-iq2-mix", "[2026-11-05T19:37:35.768Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-ab-test-readout-Nx8xpQ", "[2026-11-05T19:37:35.768Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T19:37:35.767Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "[2026-11-05T19:37:35.768Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T19:37:35.770Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T19:37:35.775Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T19:37:35.781Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-ab-test-readout-Nx8xpQ", "[2026-11-05T19:37:35.781Z] [trial] throughput-scaled ceiling: 45m → 99m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T19:37:41.502Z] [trial] daemon spawned pid=43168 port=https://127.0.0.1:51111", "[2026-11-05T19:37:42.831Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T19:37:42.847Z] [trial] meester=tamsin", "[2026-11-05T19:37:42.847Z] [trial] running scenario setup hook", "[2026-11-05T19:37:43.372Z] [trial] scenario setup complete", "[2026-11-05T19:37:43.373Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T19:37:43.418Z] [poll] initial digests hard=5d242230f8876881 soft=e2bc9971601e844f", "[2026-11-05T19:37:43.499Z] [poll] hard-progress: 5d242230f8876881 → 92136220110f6437 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T19:37:48.652Z] [poll] hard-progress: 92136220110f6437 → e6436382634ab628 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T19:38:47.944Z] [poll] hard-progress: e6436382634ab628 → 0adc992574d9b02d after 59s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=92,stream=241) sniff=1/0B)", "[2026-11-05T19:40:04.508Z] [poll] hard-progress: 0adc992574d9b02d → 29d9b9baa0c0326d after 77s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=3,slot=308,stream=1310) sniff=1/0B)", "[2026-11-05T19:51:10.458Z] [poll] hard-progress: 29d9b9baa0c0326d → 038bfcae612cc546 after 666s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=4,slot=2110,stream=7778) sniff=1/0B)", "[2026-11-05T19:51:22.423Z] [poll] hard-progress: 038bfcae612cc546 → c25bf6231d888210 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=5,slot=2132,stream=7850) sniff=1/0B)", "[2026-11-05T19:59:26.350Z] [poll] hard-progress: c25bf6231d888210 → 9a572da45a2d1390 after 484s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=7,slot=3500,stream=11917) sniff=1/0B)", "[2026-11-05T20:07:31.112Z] [poll] hard-progress: 9a572da45a2d1390 → 9c802e28f3520cd8 after 485s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=8,slot=4852,stream=15917) sniff=1/0B)", "[2026-11-05T20:07:43.073Z] [poll] hard-progress: 9c802e28f3520cd8 → 280030009ca965df after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=9,slot=4868,stream=15962) sniff=1/0B)" ] }, "metrics": { "process": { "sampleCount": 1151, "peakRssMb": 14850, "peakCpuPercent": 1737.1, "firstAtMs": 0, "lastAtMs": 6859644 }, "gpu": { "available": true, "sampleCount": 1151, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1151, "peakUsedMb": 31543, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 103039, "totalOutputTokens": 56023, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 100660, "todayTokensCached": 94145, "todayTokensOut": 9352, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 100660, "totalTokensCached": 94145, "totalTokensOut": 9352, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3516, "outputTokens": 97, "cost": 0, "durationMs": 43113, "outputTokensPerSec": 9.064258796674322, "at": "2026-11-05T21:31:59.780Z" }, "lastUpdated": "2026-11-05T21:31:59.780Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T21:32:02.938Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6861078, "meanTokensPerSec": 8.65, "genTokensPerSec": 8.65, "genTokens": 56023, "promptTokensPerSec": 376.65, "peakRssMb": 14850, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 100660, "output": 9352 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T19:37:35.763Z", "finishedAt": "2026-11-05T21:32:06.102Z", "timeToFirstArtifactMs": 145549, "timeToLastArtifactWriteMs": 6863999, "timeToFirstTokenMs": 41308, "firstTurnTtftMs": 27604, "timeToFirstToolCallMs": 69662 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 6, "completed": 5, "gateApprovals": 6, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "a-b-test-readout-eval/1", "stepId": "hypothesis-scope", "activatedAt": "2026-11-05T19:37:43.277Z", "completedAt": "2026-11-05T19:40:01.425Z", "ms": 138148, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "3d3ae99c-2248-47f2-9d77-89772306ff21" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "analyze", "activatedAt": "2026-11-05T19:40:01.426Z", "completedAt": "2026-11-05T20:14:18.049Z", "ms": 2056623, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "3d3ae99c-2248-47f2-9d77-89772306ff21" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "write", "activatedAt": "2026-11-05T20:30:06.997Z", "completedAt": "2026-11-05T21:21:53.421Z", "ms": 3106424, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "30fb088b-11df-4710-a31a-1f08c8eb8ae1", "f33a1e05-fc5f-4433-9c42-84a1330bfc75" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T21:21:53.422Z", "completedAt": "2026-11-05T21:28:45.168Z", "ms": 411746, "gateRejections": 1, "toolCalls": 26, "sessionIds": [ "834ccdd6-cea1-49ce-b2f8-7df57b7e884b", "be719aab-cb8d-4937-bee2-3eedc4eaa384" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T21:28:45.169Z", "completedAt": "2026-11-05T21:31:14.083Z", "ms": 148914, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "c57420a6-fe7b-4504-a046-236bf224c520" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T19:37:41.574Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T19:37:41.475Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 411746 }, "sessions": { "total": 7, "taskScoped": 6, "byGezel": { "dolores": 2, "nhung": 2, "noor": 1, "tolga": 2 }, "perTask": { "a-b-test-readout-eval/1": 6 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.8571428571428571, "maxMessages": 4, "maxContextFill": 0.205413818359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.205413818359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "scenarioId": "craftbook-ab-test-readout", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-05T22:11:31.190Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/readout.md", "artifactKind": "markdown", "artifactBytes": 2836, "artifactSha256": "db45f2295dfd993bdf5970a8f551ebee3ffa6500f77e918771ed5eea18dfa210", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/ab/ab-test-readout/versions/1.0.6/test.json", "axes": [ { "name": "decision rule", "description": "The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift." }, { "name": "statistical reporting", "description": "Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json." }, { "name": "honest caveats", "description": "Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action." } ], "scoreAxes": { "decision rule": 7, "statistical reporting": 2, "honest caveats": 3 }, "meanScore": 4, "justification": "The guardrail correctly blocks shipping, but the readout still says p=.4004 and nonsignificant after stats.json was patched to p=.008/true. Both retain the wrong CI−1.66 to4.16pp instead of0.33 to2.17pp. The underpower claim and bigger-sample recommendation follow that error; the claimed real refund cost is not established.", "reviewBasis": "Re-read the complete final2,836-byte readout,474-byte stats, raw CSV, authoritative brief, scope and verdict. Compared final hashes with the earlier provisional snapshot and independently computed statistics. The readout is unchanged; stats.json changed after workflow review. The declared artifact exists and is scored even though its substantive findings remain incorrect under the fixed pass.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/readout.md", "bytes": 2836, "sha256": "db45f2295dfd993bdf5970a8f551ebee3ffa6500f77e918771ed5eea18dfa210", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/data/stats.json", "bytes": 474, "sha256": "4beb58b4e2879598b5a8ef6f441ff98d3be4a5e5936c757bc45b3c342c92937c", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/data/checkout-test.csv", "bytes": 95, "sha256": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/source/experiment-brief.md", "bytes": 504, "sha256": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/scripts/ab-stats.js", "bytes": 3327, "sha256": "c777b8ac319357114d714d63a3aed1d5943e442ef8a7b965203527761e42db08", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/hypothesis-scope.md", "bytes": 1693, "sha256": "3a33bf1af55bfbe393eaaa35c6d3b9ddc3bc51abca19498512ef3152a22af1e8", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/verdict.md", "bytes": 422, "sha256": "0baab1766256270325d4ce46b3ded0ff7125088522c01eb40ad6df32cbbaa333", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:04.710Z", "reviewId": "065", "cellId": "r2-saluki-cuda-productivity-craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "scenarioId": "craftbook-ab-test-readout", "status": "scored", "axes": [ { "name": "decision rule", "description": "The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift." }, { "name": "statistical reporting", "description": "Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json." }, { "name": "honest caveats", "description": "Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-ab-test-readout", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "f9fac463d512f4fd7af64c2ed51d1947813cfa52ef8b0f8cbf9a2f181b6f91d3", "evidenceSha256": "0f1b65b4afa7cbbebf35b93e596e184f8c68ac1cc8740676275bbb59ffee036d", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/readout.md", "bytes": 2836, "sha256": "db45f2295dfd993bdf5970a8f551ebee3ffa6500f77e918771ed5eea18dfa210", "shownBytes": 2836, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/data/stats.json", "bytes": 474, "sha256": "4beb58b4e2879598b5a8ef6f441ff98d3be4a5e5936c757bc45b3c342c92937c", "shownBytes": 474, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/data/checkout-test.csv", "bytes": 95, "sha256": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "shownBytes": 95, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/source/experiment-brief.md", "bytes": 504, "sha256": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "shownBytes": 504, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/scripts/ab-stats.js", "bytes": 3327, "sha256": "c777b8ac319357114d714d63a3aed1d5943e442ef8a7b965203527761e42db08", "shownBytes": 3327, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/hypothesis-scope.md", "bytes": 1693, "sha256": "3a33bf1af55bfbe393eaaa35c6d3b9ddc3bc51abca19498512ef3152a22af1e8", "shownBytes": 1693, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/verdict.md", "bytes": 422, "sha256": "0baab1766256270325d4ce46b3ded0ff7125088522c01eb40ad6df32cbbaa333", "shownBytes": 422, "truncated": false } ], "contextFiles": [], "scoreAxes": { "decision rule": 6, "statistical reporting": 3, "honest caveats": 4 }, "justification": "Do-not-ship is correct, driven by the +0.23pp refund breach. But the readout says p=0.4004 and \"not significant\", citing stats.json, which shows p=0.008 and significant=true. So clause 2 is misreported. The CI [-1.66,4.16] matches the file but conflicts with p=0.008. Caveats are specific but rest on the false non-significance.", "limitations": "Source-only review. stats.json's CI looks wrong: recomputing gives roughly [0.33, 2.17]pp. The upstream analysis step is therefore flawed. I couldn't see why the readout's p-value differs from the file. Nothing was executed.", "meanScore": 4.333333333333333, "durationMs": 19750, "costUsd": 0.040679599999999996, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 7691, "cache_read_input_tokens": 816, "output_tokens": 983, "output_tokens_details": { "thinking_tokens": 626 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 7691, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 983, "cache_read_input_tokens": 816, "cache_creation_input_tokens": 7691, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 7691 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 983, "cacheReadInputTokens": 816, "cacheCreationInputTokens": 7691, "webSearchRequests": 0, "costUSD": 0.040679599999999996, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 626, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 7691, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-ab-test-readout / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 114m 30s; budget used: 100.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 145549`; `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":474,\"score\":19,\"scoreMax\":19,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 0 / 10 | The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 55`; `facts.timing.timeToFirstArtifactMs: 145549`; `facts.outcome.durationMs: 6870339` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 0 + 0.15 * 10 = **8.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14850 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10648 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31543 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.65 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 103039 + 56023 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 1 / 0 | `continuity.steps` |\n| Median step wall-clock | 412s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 6 / 0.86 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 21% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 145549`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":474,\"score\":19,\"scoreMax\":19,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_BOTH_SEVERE\n\nThe trial exhausted its budget while making at least thirty tool calls.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 55`\n- `facts.timing.timeToFirstArtifactMs: 145549`\n- `facts.outcome.durationMs: 6870339`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 145549`; `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":474,\"score\":19,\"scoreMax\":19,\"signals\":[],\"failReason\":null}`\n- Efficiency: The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 55`; `facts.timing.timeToFirstArtifactMs: 145549`; `facts.outcome.durationMs: 6870339`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass8/10** is unchanged. Declared advisory axes: decision rule 7/10; statistical reporting 2/10; honest caveats 3/10. Mean **4.00/10**, codex-inline/current-session.\n\nThe guardrail correctly blocks shipping, but the readout still says p=.4004 and nonsignificant after stats.json was patched to p=.008/true. Both retain the wrong CI−1.66 to4.16pp instead of0.33 to2.17pp. The underpower claim and bigger-sample recommendation follow that error; the claimed real refund cost is not established.\n\nRe-read the complete final2,836-byte readout,474-byte stats, raw CSV, authoritative brief, scope and verdict. Compared final hashes with the earlier provisional snapshot and independently computed statistics. The readout is unchanged; stats.json changed after workflow review. The declared artifact exists and is scored even though its substantive findings remain incorrect under the fixed pass.\n\n## Positives\n\n- All19 fixed checks pass and the workflow eventually completes five stages. The no-ship decision correctly honors refund0.80%→1.03%, exceeding the0.20pp limit by0.03pp; purchase rates12.00%/13.25%, lift1.25pp and relative10.42% are correct (outcome.success; sniff.latest; continuity.steps.completed=5; readout and raw CSV).\n- The final standard feedback repair corrected stats.json p_value to0.008 and significant_at_95 totrue. It was performed by the model through replace_in_file at21:31:59Z, not by the monitor (toolUse.byTool.replace_in_file=1; recording/chat-events.jsonl; log.txt).\n\n## Negatives\n\n- The readout still reports p=.4004, a nonsignificant conversion result and CI[-1.66,4.16]pp. Correct pooled z=2.66125, two-sided p=.007785, and unpooled Wald95% CI[.32954,2.17046]pp establish significance under the named method. Final stats.json retains the wrong CI and now contradicts the readout on significance (sniff.latest full fixed pass; artifacts.otherFileCount=8; advisory-statistical-audit.json).\n- The late repair changes only p and its Boolean. The evaluator explicitly prompted with z≈2.66/p≈.008; it does not validate the CI value or cross-file semantic consistency. The earlier all-PASS reviewer verdict is stale after the repair and was already wrong about the calculation. A fixed pass must not be interpreted as a correct decision-ready report (outcome.reason; toolUse.byTool.replace_in_file=1; log.txt at21:31:15Z; final verdict/readout/stats).\n- run_installed_script(node) failed because node was not an installed project script; the model then wrote stats directly. Its unused helper separately has an invalid normal CDF (CDF(0)=.8403; p≈.01662 for this dataset). Manual stats also disagree with that helper (toolUse.byTool.run_installed_script=1 and write_file=2; advisory-tool-evidence.json; preserved helper and independent audit).\n- An advance before saving verdict.md caused the evaluate gate to reject and route back to write. The stale reviewer continued repeated denied writes, a denied pause and read/task-note traffic for roughly49minutes before asking a question and yielding. Its notes told the next writer all substantive criteria passed. Standard scheduling resumed and completed; no monitor intervention occurred (continuity.steps.gateRejections=1; toolUse.byTool.write_artifact=8, set_task_status=1, ask_user_question=1; recording/chat-events.jsonl20:30–21:19Z). autoAnswer.total=0, so no automatic answer is claimed.\n- The underpowered/bigger-sample advice follows the wrong CI, without a prespecified minimum effect or power target. A deterministic guardrail breach is not proof that refund harm is statistically established.55calls and budgetUsedFraction1 reduce fixed efficiency to0 (toolUse.totalToolCalls; outcome.budgetUsedFraction; score.axes.efficiency; final readout).\n\n## Performance and GPU fit\n\nElapsed **114.51min**,55 calls;56023 recorded generated tokens at **8.65tok/s**, prefill376.65tok/s. Peak VRAM **10648/12227MiB**; process-tree RSS **14850MiB**; whole-system RAM **31543/31865MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications; shared GPU memory and paging were not measured.\n\nOne native0.1.57 CUDA launch,65,536-token allocation, one slot, f16KV, reasoning4096. Dense FFNs for39/64 layers on GPU and25 on CPU: hybrid residency. No failed endpoint, explicit OOM or context overflow found. Maximum captured session fill20.5%; maximum observed native released-slot size23985tokens (36.6%). Neither is a continuous KV peak. No mid-turn compactions. The standard runner extended its ceiling twice by15minutes under its existing hard-progress policy (log.txt21:16:40Z and21:31:38Z); no timeout tuning was changed. The readout’s initial p/CI error was not a GPU or native engine failure.\n\nPerformance is separate from fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: in Gilde ab-test-readout craftbook.json based on1.0.6, steps[id=analyze].prompt, require method, standard error and an execution receipt or explicit “not executed” label for derived figures. In steps[id=evaluate].prompt add: “Recompute one derived value from raw inputs; consistency with an intermediate file alone is insufficient. If an upstream result changes, recheck every dependent assertion.” Do not encode this fixture’s p-value or expected decision in model prompts.\n\n## Strategic experiment — not applied\n\nProposed experiments, not applied: (1) packages/core/src/local-loop/llama-cpp-session.ts and packages/service/src/providers/mcp-bridge.ts should treat confirmed stale_task_step ownership loss as a terminal turn signal, preserving the write guard while allowing the newly active step to run. (2) packages/service/src/tasks/step-gate.ts should invalidate approval of dependent deliverables when a consumed result changes. (3) test a reusable numeric-invariant check in packages/core/src/checks that recomputes rate uncertainty and cross-checks p, CI and significance; do not just add another expected fixture literal.\n\n## Generalization and next experiment\n\nR1 fixed9/advisory6.67 had correct figures; R2 fixed8/advisory4 has wrong CI and a stale readout after partial repair. Same frozen native/tuning identity: a repetition regression, not a native-version effect. Cross-check turn termination against the unrelated spreadsheet repair loop and numeric/execution handling against records-intake/data-normalization. Measure stale-step calls, derived-value accuracy, artifact consistency and completion before calling a proposal a general fix.\n\nRun any proposed experiment separately after this frozen campaign. No source, prompt, gate, tuning, model file or campaign state was changed, and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/facts.json): pass, fixed9, 80.47min, 2.74tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/facts.json): pass, fixed8.5, 4.80min, 124.5tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T09-49-13-291Z-632u/facts.json): fail, fixed1.2, 79.63min, 116.94tok/s.\n\nOther-model/native-version history is contextual, not a controlled comparison.\n\n## Strategic note\n\nThe final pass demonstrates completion of the checked workflow predicates, not reliable statistical synthesis. This is a clear quality regression against the first repetition and a concrete example of a local repair leaving a dependent artifact stale. No stronger-model recommendation follows from it.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json, historical-comparison.json, daemon.log and the preserved final deliverables. Also see advisory-statistical-audit.json and both earlier provisional snapshots.\n" }, { "cellId": "r2-saluki-cuda-productivity-craftbook-annotated-bibliography", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-annotated-bibliography", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T21:35:05.920Z", "finishedAt": "2026-11-05T21:50:25.159Z", "durationMs": 919239, "success": true, "reason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 110782 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-annotated-bibliography", "bytes": 6357, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.15490879272323563 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 19 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 110782 }, { "source": "facts", "path": "outcome.durationMs", "value": 919239 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 154, "peakRssMb": 14372, "peakCpuPercent": 1214.6, "firstAtMs": 0, "lastAtMs": 906503 }, "gpu": { "available": true, "sampleCount": 154, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 154, "peakUsedMb": 31548, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 50828, "totalOutputTokens": 6142, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 43796, "todayTokensCached": 42385, "todayTokensOut": 2668, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 43796, "totalTokensCached": 42385, "totalTokensOut": 2668, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 12267, "cachedInputTokens": 12197, "outputTokens": 379, "cost": 0, "durationMs": 227715, "outputTokensPerSec": 8.363196545583879, "at": "2026-11-05T21:48:47.387Z" }, "lastUpdated": "2026-11-05T21:48:47.387Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T21:50:20.208Z" } }, "source": "engine-log" }, "derived": { "durationMs": 908027, "meanTokensPerSec": 8.46, "genTokensPerSec": 8.46, "genTokens": 6142, "promptTokensPerSec": 399.75, "peakRssMb": 14372, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43796, "output": 2668 } } } }, "craftbook": { "craftbookId": "annotated-bibliography", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 4, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "annotated-bibliography-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T21:35:13.391Z", "completedAt": "2026-11-05T21:36:56.787Z", "ms": 103396, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "ecbf5220-c571-4ba6-a268-707dfa458b0a" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "research", "activatedAt": "2026-11-05T21:36:56.788Z", "completedAt": "2026-11-05T21:41:01.440Z", "ms": 244652, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "7e65ca3a-316c-463e-aa0d-6e1f83d33300" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "write", "activatedAt": "2026-11-05T21:41:01.442Z", "completedAt": "2026-11-05T21:44:53.523Z", "ms": 232081, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "65762ad6-f4e0-4163-9de4-3b296d6002d1" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T21:44:53.524Z", "completedAt": "2026-11-05T21:48:47.546Z", "ms": 234022, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "13d6b20e-72c4-40df-bffb-c3e83f009d9e" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T21:48:47.548Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "7ca2736e-b26e-4f3e-9478-5f1b9f5ae3fa" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T21:35:11.852Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T21:35:11.768Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 233051.5 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "ashot": 1, "asif": 1, "bente": 1, "leonardo": 1, "vaiana": 1 }, "perTask": { "annotated-bibliography-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.15985107421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.15985107421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/annotated-bibliography", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/reading-list", "knowledge://bendyline/handboek/gezel-app-page-style-guide", "knowledge://bendyline/handboek/craftbook/cohort-analysis" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "durationMs": 919239, "timeoutMs": 5934066, "budgetUsedFraction": 0.15490879272323563 }, "timing": { "startedAt": "2026-11-05T21:35:05.920Z", "finishedAt": "2026-11-05T21:50:25.159Z", "timeToFirstArtifactMs": 110782, "timeToLastArtifactWriteMs": 821467, "timeToFirstTokenMs": 34857, "firstTurnTtftMs": 20912, "timeToFirstToolCallMs": 41237 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 19, "byTool": { "read_file": 5, "write_artifact": 3, "read_artifact": 5, "write_file": 1, "write_task_note": 2, "advance_task_step": 1, "validate": 1, "set_task_status": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 7691, "filePath": "craftbook-annotated-bibliography", "bytes": 0, "score": 0, "scoreMax": 9, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/source-cards.md" }, { "atMs": 42424, "filePath": "craftbook-annotated-bibliography", "bytes": 0, "score": 1, "scoreMax": 9, "signals": [], "failReason": "task sourced from craftbook annotated-bibliography has not finished: its gates have not accepted the result yet" }, { "atMs": 563269, "filePath": "craftbook-annotated-bibliography", "bytes": 6357, "score": 8, "scoreMax": 9, "signals": [], "failReason": "task sourced from craftbook annotated-bibliography has not finished: its gates have not accepted the result yet" }, { "atMs": 913773, "filePath": "craftbook-annotated-bibliography", "bytes": 6357, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-annotated-bibliography", "bytes": 6357, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T21:35:05.925Z] [trial] id=craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0 scenario=craftbook-annotated-bibliography model=underdog-saluki-27b-iq2-mix", "[2026-11-05T21:35:05.925Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "[2026-11-05T21:35:05.925Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T21:35:05.925Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-annotated-bibliography-Rx2W3G", "[2026-11-05T21:35:05.927Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T21:35:05.925Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T21:35:05.932Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T21:35:05.937Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-annotated-bibliography-Rx2W3G", "[2026-11-05T21:35:05.938Z] [trial] throughput-scaled ceiling: 45m → 99m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T21:35:11.768Z] [trial] daemon spawned pid=32584 port=https://127.0.0.1:62472", "[2026-11-05T21:35:12.994Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T21:35:13.011Z] [trial] running scenario setup hook", "[2026-11-05T21:35:13.011Z] [trial] meester=yusuf", "[2026-11-05T21:35:13.536Z] [trial] scenario setup complete", "[2026-11-05T21:35:13.537Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T21:35:13.577Z] [poll] initial digests hard=7813263928eb75cf soft=4a18c54f1ff42372", "[2026-11-05T21:35:13.645Z] [poll] hard-progress: 7813263928eb75cf → e17d60f41508b0e9 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T21:35:18.826Z] [poll] hard-progress: e17d60f41508b0e9 → 7bd4fbca884b5cf3 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-05T21:35:48.382Z] [poll] hard-progress: 7bd4fbca884b5cf3 → 4228a8ea1b5bc8f2 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=24,stream=43) sniff=1/0B)", "[2026-11-05T21:36:59.101Z] [poll] hard-progress: 4228a8ea1b5bc8f2 → 6073dc8cdc13de61 after 71s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=206,stream=956) sniff=1/0B)", "[2026-11-05T21:37:46.766Z] [poll] hard-progress: 6073dc8cdc13de61 → 49cc2ae1b9524682 after 48s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=242,stream=1037) sniff=1/0B)", "[2026-11-05T21:41:07.460Z] [poll] hard-progress: 49cc2ae1b9524682 → 2ed472f22b57a4ff after 201s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=5,slot=782,stream=4151) sniff=1/0B)", "[2026-11-05T21:41:37.005Z] [poll] hard-progress: 2ed472f22b57a4ff → 2a00cff0f50adb0a after 30s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=7,slot=802,stream=4191) sniff=1/0B)", "[2026-11-05T21:41:48.867Z] [poll] hard-progress: 2a00cff0f50adb0a → af52ab7367f5c7b2 after 12s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=8,slot=814,stream=4218) sniff=1/0B)", "[2026-11-05T21:44:29.249Z] [poll] hard-progress: af52ab7367f5c7b2 → fbdeef27e59f27e5 after 160s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=9,slot=1257,stream=6833) sniff=8/6357B)" ] }, "metrics": { "process": { "sampleCount": 154, "peakRssMb": 14372, "peakCpuPercent": 1214.6, "firstAtMs": 0, "lastAtMs": 906503 }, "gpu": { "available": true, "sampleCount": 154, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 154, "peakUsedMb": 31548, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 50828, "totalOutputTokens": 6142, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 43796, "todayTokensCached": 42385, "todayTokensOut": 2668, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 43796, "totalTokensCached": 42385, "totalTokensOut": 2668, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 12267, "cachedInputTokens": 12197, "outputTokens": 379, "cost": 0, "durationMs": 227715, "outputTokensPerSec": 8.363196545583879, "at": "2026-11-05T21:48:47.387Z" }, "lastUpdated": "2026-11-05T21:48:47.387Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T21:50:20.208Z" } }, "source": "engine-log" }, "derived": { "durationMs": 908027, "meanTokensPerSec": 8.46, "genTokensPerSec": 8.46, "genTokens": 6142, "promptTokensPerSec": 399.75, "peakRssMb": 14372, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43796, "output": 2668 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T21:35:05.920Z", "finishedAt": "2026-11-05T21:50:25.159Z", "timeToFirstArtifactMs": 110782, "timeToLastArtifactWriteMs": 821467, "timeToFirstTokenMs": 34857, "firstTurnTtftMs": 20912, "timeToFirstToolCallMs": 41237 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 4, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "annotated-bibliography-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T21:35:13.391Z", "completedAt": "2026-11-05T21:36:56.787Z", "ms": 103396, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "ecbf5220-c571-4ba6-a268-707dfa458b0a" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "research", "activatedAt": "2026-11-05T21:36:56.788Z", "completedAt": "2026-11-05T21:41:01.440Z", "ms": 244652, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "7e65ca3a-316c-463e-aa0d-6e1f83d33300" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "write", "activatedAt": "2026-11-05T21:41:01.442Z", "completedAt": "2026-11-05T21:44:53.523Z", "ms": 232081, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "65762ad6-f4e0-4163-9de4-3b296d6002d1" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T21:44:53.524Z", "completedAt": "2026-11-05T21:48:47.546Z", "ms": 234022, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "13d6b20e-72c4-40df-bffb-c3e83f009d9e" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T21:48:47.548Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "7ca2736e-b26e-4f3e-9478-5f1b9f5ae3fa" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T21:35:11.852Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T21:35:11.768Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 233051.5 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "ashot": 1, "asif": 1, "bente": 1, "leonardo": 1, "vaiana": 1 }, "perTask": { "annotated-bibliography-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.15985107421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.15985107421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "scenarioId": "craftbook-annotated-bibliography", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-05T22:11:31.636Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/workspace/annotated-bibliography-eval/bibliography.md", "artifactKind": "markdown", "artifactBytes": 6375, "artifactSha256": "5d859afc3c4cf138476a201dcc25808e123acedbc525dc37fb60c2134bf31dd4", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/an/annotated-bibliography/versions/1.0.4/test.json", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ], "scoreAxes": { "accuracy": 8, "grounding": 4, "structure": 8, "actionability": 5 }, "meanScore": 6.25, "justification": "All six citations and supplied figures are preserved in clear, consistent annotations. Unsupported conclusions remain: omitted outcomes become unmeasured outcomes, coaching becomes proof technology alone is insufficient, and cost feasibility becomes evidence mixed modalities are necessary. Relevance is clear, but follow-up actions remain general.", "reviewBasis": "Read all6,375 bytes of the final bibliography, the full1,294-byte offline source-card fixture,1,679-byte scope,7,109-byte source packet and1,529-byte verdict. Assessed fidelity to the supplied cards, not external-study truth or clinical advice. The trial contains one successful bibliography write and a file-non-empty validation only.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/workspace/annotated-bibliography-eval/bibliography.md", "bytes": 6375, "sha256": "5d859afc3c4cf138476a201dcc25808e123acedbc525dc37fb60c2134bf31dd4", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/workspace/annotated-bibliography-eval/source/source-cards.md", "bytes": 1294, "sha256": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/scope.md", "bytes": 1679, "sha256": "64af78f5ab97e1434ead970e0395182896a9aea072f86acda070e0e1597148ec", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/sources.md", "bytes": 7109, "sha256": "27c12b61df547982f4b7f24f23a861f0f7b4f1736d7639fc4bae748ac44c541b", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/verdict.md", "bytes": 1529, "sha256": "42c0106e9c4085dd926746b433c4f0c351cc5fd7acc77b89635829af3c248ee4", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:06.145Z", "reviewId": "066", "cellId": "r2-saluki-cuda-productivity-craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "scenarioId": "craftbook-annotated-bibliography", "status": "scored", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-annotated-bibliography", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "9bc18621a477f47a33c021ae378036d0a53657bfee6d16a1ebc6400de00ddee4", "evidenceSha256": "c8e6564161dc54522f8ed13259f73cdf4fa7ea2fd062eed394247575bae59180", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/workspace/annotated-bibliography-eval/bibliography.md", "bytes": 6375, "sha256": "5d859afc3c4cf138476a201dcc25808e123acedbc525dc37fb60c2134bf31dd4", "shownBytes": 6375, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/workspace/annotated-bibliography-eval/source/source-cards.md", "bytes": 1294, "sha256": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "shownBytes": 1294, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/scope.md", "bytes": 1679, "sha256": "64af78f5ab97e1434ead970e0395182896a9aea072f86acda070e0e1597148ec", "shownBytes": 1679, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/sources.md", "bytes": 7109, "sha256": "27c12b61df547982f4b7f24f23a861f0f7b4f1736d7639fc4bae748ac44c541b", "shownBytes": 7109, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/verdict.md", "bytes": 1529, "sha256": "42c0106e9c4085dd926746b433c4f0c351cc5fd7acc77b89635829af3c248ee4", "shownBytes": 1529, "truncated": false } ], "contextFiles": [], "scoreAxes": { "accuracy": 8, "grounding": 7, "structure": 8, "actionability": 6 }, "justification": "All six entries match the seeded cards on figures (11%, 72h, 900, n=240, 18 points, 42, 1,600). Ordering and APA format are consistent, and each entry has three labeled parts. Small embellishments: Williams \"offers a privacy checklist\" and \"2019 literature\", Chen \"rural\", and Allen \"strongest evidence\". Relevance notes are usable.", "limitations": "Axes are generic and fit a bibliography loosely. Only the cards were checked. Actual article existence and APA details such as volume and DOI were not verified. The scope and sources files were reviewed only briefly. Runtime gate behaviour was not observed.", "meanScore": 7.25, "durationMs": 17598, "costUsd": 0.046018, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 9615, "cache_read_input_tokens": 840, "output_tokens": 747, "output_tokens_details": { "thinking_tokens": 391 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 9615, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 747, "cache_read_input_tokens": 840, "cache_creation_input_tokens": 9615, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9615 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 747, "cacheReadInputTokens": 840, "cacheCreationInputTokens": 9615, "webSearchRequests": 0, "costUSD": 0.046018, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 391, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 9615, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-annotated-bibliography / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 15m 19s; budget used: 15.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 110782`; `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":6357,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.15490879272323563`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 110782`; `facts.outcome.durationMs: 919239` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14372 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31548 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.46 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 50828 + 6142 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 4 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 233s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.71 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 16% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 110782`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":6357,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.15490879272323563`\n- `facts.toolUse.totalToolCalls: 19`\n- `facts.timing.timeToFirstArtifactMs: 110782`\n- `facts.outcome.durationMs: 919239`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 110782`; `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":6357,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.15490879272323563`; `facts.toolUse.totalToolCalls: 19`; `facts.timing.timeToFirstArtifactMs: 110782`; `facts.outcome.durationMs: 919239`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed result **pass9.5/10** is unchanged. Declared advisory axes: accuracy 8/10; grounding 4/10; structure 8/10; actionability 5/10. Mean **6.25/10**, codex-inline/current-session.\n\nAll six citations and supplied figures are preserved in clear, consistent annotations. Unsupported conclusions remain: omitted outcomes become unmeasured outcomes, coaching becomes proof technology alone is insufficient, and cost feasibility becomes evidence mixed modalities are necessary. Relevance is clear, but follow-up actions remain general.\n\nRead all6,375 bytes of the final bibliography, the full1,294-byte offline source-card fixture,1,679-byte scope,7,109-byte source packet and1,529-byte verdict. Assessed fidelity to the supplied cards, not external-study truth or clinical advice. The trial contains one successful bibliography write and a file-non-empty validation only.\n\n## Positives\n\n- All9 fixed checks pass. Six entries preserve author, year, title and journal metadata plus supplied quantities:72h/30days/11%/four clinics;900 households; n240/18points;42 interviews;1,600 annual encounters (outcome.success; sniff.latest; final bibliography versus source/source-cards.md). No missing DOI, issue or volume was fabricated.\n- Alphabetical entries use consistent Summary/Critical evaluation/Relevance sections. Several caveats are useful and explicitly bounded: observational design, self-reported bandwidth, single-region pilot, cost-model assumptions absent from the card, and privacy evidence adjacent to the question (artifacts.otherFileCount=6; toolUse.byTool.read_file=5; final bibliography).\n\n## Negatives\n\n- The bibliography asserts Chen does not isolate provider versus infrastructure factors and Dunn does not capture readmission/cost/experience outcomes, though the cards merely omit those details. It extrapolates coaching efficacy to technology being insufficient and cost break-even to clinical need for mixed modalities. Williams being useful for a checklist becomes offering a checklist (sniff.latest full pass; bibliography versus source cards; artifacts.otherFileCount=6).\n- The scope/source packet labels all six entries peer-reviewed without verification in the seeded cards; research notes propagate interpretive prose to the writer. The reviewer signs off on no fabrication by checking citation metadata while missing unsupported method/causal claims (toolUse.byTool.write_artifact=3; sources.md, scope.md and verdict.md).\n- validate only checks file-non-empty, not citation entailment or methodological support. Root status is set complete while the captured finish step has no completedAt; do not equate four recorded completed stages with five gate-completed stages (toolUse.byTool.validate=1, set_task_status=1; continuity.steps.completed=4; continuity.steps.perStep[finish]). This does not alter the fixed terminal success.\n\n## Performance and GPU fit\n\nElapsed **15.32min**,19 calls;6142 recorded generated tokens at **8.46tok/s**, prefill399.75tok/s. Peak VRAM **10640/12227MiB**; process-tree RSS **14372MiB**; whole-system RAM **31548/31865MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system RAM includes other applications; shared GPU memory and paging were not measured.\n\nOne native0.1.57 CUDA launch,65,536-token allocation, one slot, f16KV, reasoning4096. Dense FFNs for39/64 layers on GPU and25 on CPU: hybrid residency. No failed endpoint, explicit OOM or context overflow found. Maximum captured session fill16.0%; maximum observed native released-slot size14415tokens (22.0%). Neither is a continuous KV peak. No mid-turn compactions. The final root-completion tool succeeds; there is no observed engine restart or memory failure. The limitation is source fidelity rather than ability to write or load the model.\n\nPerformance is separate from fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: in the Gilde annotated-bibliography craftbook.json based on1.0.4, steps[id=research].prompt and steps[id=evaluate].prompt, add: “Distinguish not reported in the card from not measured in the study. Mark methodological extrapolations as hypotheses. Do not assert peer review, comparison arms or omitted outcomes without explicit source evidence.” Measure unsupported-claim count while preserving genuinely useful critical analysis.\n\n## Strategic experiment — not applied\n\nProposed experiment, not applied: extend packages/service/src/chat/evidence-ledger.ts with source-kind and claim-support spans carried through research handoffs. In packages/core/src/checks/claims.ts, distinguish bibliographic metadata agreement from support for methods, outcomes and causal statements. Intermediate notes should remain derived evidence, not upgrade a weak card into verified study details.\n\n## Generalization and next experiment\n\nR1 fixed9.5/advisory6.25 and R2 fixed9.5/advisory6.25 both preserve citation structure while adding unsupported methodology. Compare any source-boundary experiment against the unrelated Wikipedia brief and operational Word report, which also contain unsupported explanations under fixed passes. Measure facts preserved, unsupported claims and false write rejections; no content prompt should memorize these authors or statistics.\n\nRun any proposed experiment separately after this frozen campaign. No source, prompt, gate, tuning, model file or campaign state was changed, and no capability failure was rerun.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/facts.json): pass, fixed9.5, 21.22min, 8.96tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/facts.json): pass, fixed9.5, 1.78min, 122.68tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-16-32-692Z-xwts/facts.json): pass, fixed9, 3.01min, 120.44tok/s.\n\nOther-model/native-version history is contextual, not a controlled comparison.\n\n## Strategic note\n\nThe recurring issue is provenance inflation through research notes and reviews. The same fixed score across repetitions does not remove this advisory limitation. These seeded offline citations were not independently verified publications, and no clinical recommendation is made.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json, historical-comparison.json, daemon.log and the preserved final deliverables.\n" }, { "cellId": "r2-saluki-cuda-productivity-craftbook-research-to-document", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-research-to-document", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T21:53:08.914Z", "finishedAt": "2026-11-05T22:18:58.685Z", "durationMs": 1549771, "success": true, "reason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 142136 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-research-to-document", "bytes": 1951, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2611651100611284 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 43 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 142136 }, { "source": "facts", "path": "outcome.durationMs", "value": 1549771 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 260, "peakRssMb": 15346, "peakCpuPercent": 1208.3, "firstAtMs": 0, "lastAtMs": 1537478 }, "gpu": { "available": true, "sampleCount": 260, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 260, "peakUsedMb": 31409, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 107033, "totalOutputTokens": 9155, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 102373, "todayTokensCached": 101297, "todayTokensOut": 1387, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 102373, "totalTokensCached": 101297, "totalTokensOut": 1387, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9758, "cachedInputTokens": 9689, "outputTokens": 48, "cost": 0, "durationMs": 92817, "outputTokensPerSec": 8.439965715781819, "at": "2026-11-05T22:18:51.258Z" }, "lastUpdated": "2026-11-05T22:18:51.258Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T22:18:53.942Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1538841, "meanTokensPerSec": 8.14, "genTokensPerSec": 8.14, "genTokens": 9155, "promptTokensPerSec": 357.39, "peakRssMb": 15346, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 102373, "output": 1387 } } } }, "craftbook": { "craftbookId": "research-to-document", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "word-document-from-content-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T21:53:16.435Z", "completedAt": "2026-11-05T21:55:31.224Z", "ms": 134789, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "7200bcb4-ec0b-4fee-8766-4845d5978d3c" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "sources", "activatedAt": "2026-11-05T21:55:31.226Z", "completedAt": "2026-11-05T21:57:28.020Z", "ms": 116794, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "eebb3cb4-4a59-4ec9-a2ad-f098917e19d2" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "write", "activatedAt": "2026-11-05T21:57:28.022Z", "completedAt": "2026-11-05T22:01:05.989Z", "ms": 217967, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "0e04839d-5636-459c-b89e-4831d77ff1bb" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "publish", "activatedAt": "2026-11-05T22:01:05.996Z", "completedAt": "2026-11-05T22:04:51.807Z", "ms": 225811, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "0e04839d-5636-459c-b89e-4831d77ff1bb" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T22:04:51.808Z", "completedAt": "2026-11-05T22:17:13.348Z", "ms": 741540, "gateRejections": 0, "toolCalls": 14, "sessionIds": [ "a6319b37-3371-4ef2-bfe3-45d24bd34665" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T22:17:13.349Z", "completedAt": "2026-11-05T22:18:51.216Z", "ms": 97867, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "dca1e6ba-94d1-48e5-bf9b-3123e7770d1e" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T21:53:14.703Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T21:53:14.601Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 176378 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "aicha": 2, "caroline": 1, "gianluca": 1, "jun": 1, "priya": 1, "urantsetseg": 1 }, "perTask": { "word-document-from-content-eval/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.3113250732421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3113250732421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/project-type/support-desk", "knowledge://bendyline/handboek/craftbook/morning-brief", "knowledge://bendyline/handboek/craftbook/source-quality-audit", "knowledge://bendyline/handboek/project-type/hiring-desk", "knowledge://bendyline/handboek/project-type/vendor-contracts" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "durationMs": 1549771, "timeoutMs": 5934066, "budgetUsedFraction": 0.2611651100611284 }, "timing": { "startedAt": "2026-11-05T21:53:08.914Z", "finishedAt": "2026-11-05T22:18:58.685Z", "timeToFirstArtifactMs": 142136, "timeToLastArtifactWriteMs": 477111, "timeToFirstTokenMs": 32801, "firstTurnTtftMs": 19437, "timeToFirstToolCallMs": 39683 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Planner", "Researcher", "Copywriter", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 43, "byTool": { "read_file": 8, "list_dir": 2, "read_task_notes": 4, "write_artifact": 2, "read_artifact": 6, "write_file": 2, "write_task_note": 4, "advance_task_step": 4, "list_roots": 2, "convert_document": 1, "inspect_document": 2, "preview_document": 2, "save_artifact": 1, "stat": 1, "validate": 1, "list_artifacts": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [ { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_215730_word-document-docblocks-artifact-surface/tool-0-img-0.png", "bytes": 19310 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_215730_word-document-docblocks-artifact-surface/tool-0-img-1.png", "bytes": 92842 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_215730_word-document-docblocks-artifact-surface/tool-0-img-2.png", "bytes": 37889 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_215730_word-document-docblocks-artifact-surface/tool-0-img-3.png", "bytes": 108366 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_220455_new-session/tool-0-img-0.png", "bytes": 19310 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_220455_new-session/tool-0-img-1.png", "bytes": 92842 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_220455_new-session/tool-0-img-2.png", "bytes": 37889 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_220455_new-session/tool-0-img-3.png", "bytes": 108366 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-05_220455_new-session/tool-0-img-4.png", "bytes": 116028 } ], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 7765, "filePath": "craftbook-research-to-document", "bytes": 0, "score": 1, "scoreMax": 14, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/brief.md" }, { "atMs": 42572, "filePath": "craftbook-research-to-document", "bytes": 0, "score": 2, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 455985, "filePath": "craftbook-research-to-document", "bytes": 1951, "score": 8, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 574892, "filePath": "craftbook-research-to-document", "bytes": 1951, "score": 9, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 592625, "filePath": "craftbook-research-to-document", "bytes": 1951, "score": 10, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 616220, "filePath": "craftbook-research-to-document", "bytes": 1951, "score": 11, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 645608, "filePath": "craftbook-research-to-document", "bytes": 1951, "score": 13, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 1544608, "filePath": "craftbook-research-to-document", "bytes": 1951, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-research-to-document", "bytes": 1951, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T21:53:08.917Z] [trial] id=craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0 scenario=craftbook-research-to-document model=underdog-saluki-27b-iq2-mix", "[2026-11-05T21:53:08.917Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-research-to-document-rzeM7Y", "[2026-11-05T21:53:08.917Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T21:53:08.917Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "[2026-11-05T21:53:08.917Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T21:53:08.920Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T21:53:08.923Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T21:53:08.929Z] [trial] throughput-scaled ceiling: 45m → 99m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T21:53:08.929Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-research-to-document-rzeM7Y", "[2026-11-05T21:53:14.655Z] [trial] daemon spawned pid=25048 port=https://127.0.0.1:61561", "[2026-11-05T21:53:16.009Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T21:53:16.030Z] [trial] meester=priya", "[2026-11-05T21:53:16.030Z] [trial] running scenario setup hook", "[2026-11-05T21:53:16.580Z] [trial] scenario setup complete", "[2026-11-05T21:53:16.582Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T21:53:16.626Z] [poll] initial digests hard=7c71fe6340e58f36 soft=302be5c73ae9e229", "[2026-11-05T21:53:16.718Z] [poll] hard-progress: 7c71fe6340e58f36 → d813665cfc55eeb3 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T21:53:21.857Z] [poll] hard-progress: d813665cfc55eeb3 → f5568ad283980ecf after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T21:53:51.551Z] [poll] hard-progress: f5568ad283980ecf → 4d931b4f8099d5af after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=31,stream=62) sniff=2/0B)", "[2026-11-05T21:54:03.310Z] [poll] hard-progress: 4d931b4f8099d5af → 7efc27ad97511caa after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=52,stream=127) sniff=2/0B)", "[2026-11-05T21:54:09.168Z] [poll] hard-progress: 7efc27ad97511caa → f2b3b1300a2cd090 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=61,stream=150) sniff=2/0B)", "[2026-11-05T21:55:31.635Z] [poll] hard-progress: f2b3b1300a2cd090 → bcc76ab9601b000b after 82s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=281,stream=1213) sniff=2/0B)", "[2026-11-05T21:55:37.404Z] [poll] hard-progress: bcc76ab9601b000b → 67a0e0b04880c218 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=281,stream=1213) sniff=2/0B)", "[2026-11-05T21:56:18.768Z] [poll] hard-progress: 67a0e0b04880c218 → 78da1d0a34e94c55 after 41s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=321,stream=1309) sniff=2/0B)", "[2026-11-05T21:56:36.472Z] [poll] hard-progress: 78da1d0a34e94c55 → 76de8e1ab22a0bf8 after 18s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=360,stream=1427) sniff=2/0B)" ] }, "metrics": { "process": { "sampleCount": 260, "peakRssMb": 15346, "peakCpuPercent": 1208.3, "firstAtMs": 0, "lastAtMs": 1537478 }, "gpu": { "available": true, "sampleCount": 260, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 260, "peakUsedMb": 31409, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 107033, "totalOutputTokens": 9155, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 102373, "todayTokensCached": 101297, "todayTokensOut": 1387, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 102373, "totalTokensCached": 101297, "totalTokensOut": 1387, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9758, "cachedInputTokens": 9689, "outputTokens": 48, "cost": 0, "durationMs": 92817, "outputTokensPerSec": 8.439965715781819, "at": "2026-11-05T22:18:51.258Z" }, "lastUpdated": "2026-11-05T22:18:51.258Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T22:18:53.942Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1538841, "meanTokensPerSec": 8.14, "genTokensPerSec": 8.14, "genTokens": 9155, "promptTokensPerSec": 357.39, "peakRssMb": 15346, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 102373, "output": 1387 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T21:53:08.914Z", "finishedAt": "2026-11-05T22:18:58.685Z", "timeToFirstArtifactMs": 142136, "timeToLastArtifactWriteMs": 477111, "timeToFirstTokenMs": 32801, "firstTurnTtftMs": 19437, "timeToFirstToolCallMs": 39683 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "word-document-from-content-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T21:53:16.435Z", "completedAt": "2026-11-05T21:55:31.224Z", "ms": 134789, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "7200bcb4-ec0b-4fee-8766-4845d5978d3c" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "sources", "activatedAt": "2026-11-05T21:55:31.226Z", "completedAt": "2026-11-05T21:57:28.020Z", "ms": 116794, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "eebb3cb4-4a59-4ec9-a2ad-f098917e19d2" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "write", "activatedAt": "2026-11-05T21:57:28.022Z", "completedAt": "2026-11-05T22:01:05.989Z", "ms": 217967, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "0e04839d-5636-459c-b89e-4831d77ff1bb" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "publish", "activatedAt": "2026-11-05T22:01:05.996Z", "completedAt": "2026-11-05T22:04:51.807Z", "ms": 225811, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "0e04839d-5636-459c-b89e-4831d77ff1bb" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T22:04:51.808Z", "completedAt": "2026-11-05T22:17:13.348Z", "ms": 741540, "gateRejections": 0, "toolCalls": 14, "sessionIds": [ "a6319b37-3371-4ef2-bfe3-45d24bd34665" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "finish", "activatedAt": "2026-11-05T22:17:13.349Z", "completedAt": "2026-11-05T22:18:51.216Z", "ms": 97867, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "dca1e6ba-94d1-48e5-bf9b-3123e7770d1e" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T21:53:14.703Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T21:53:14.601Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 176378 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "aicha": 2, "caroline": 1, "gianluca": 1, "jun": 1, "priya": 1, "urantsetseg": 1 }, "perTask": { "word-document-from-content-eval/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.3113250732421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3113250732421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-productivity-craftbook-research-to-document", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "scenarioId": "craftbook-research-to-document", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-05T23:08:01.283Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/workspace/word-document-from-content-eval/documents/eval/report.md", "artifactBytes": 1955, "artifactSha256": "a5b472f4a0c1e1860d5ee3ce070b57063773b94901a14b7178aee84f11c482a1", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/re/research-to-document/versions/1.2.8/test.json", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ], "scoreAxes": { "grounding": 4, "structure": 7, "convertibility": 8 }, "meanScore": 6.33, "justification": "All supplied figures and actions survive in organized Markdown and a real DOCX with a table. Unsupported claims make status silence the biggest repeat-contact driver, say barcode exceptions stall returns, and attribute specific email triggers to the brief. The reviewer explicitly accepts the causal overstatement. Conversion succeeds; source fidelity is weaker.", "reviewBasis": "Re-read the final 1,955-byte Markdown and 405-byte brief. Final source and 19,240-byte DOCX hashes match the previously reviewed snapshot exactly. ZIP/XML and all five recorded reconstructed-import previews were already inspected; they establish content/structure, not Word-native pagination. Read the final review note and terminal step receipt. No new conversion or model run was performed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/workspace/word-document-from-content-eval/documents/eval/report.md", "bytes": 1955, "sha256": "a5b472f4a0c1e1860d5ee3ce070b57063773b94901a14b7178aee84f11c482a1", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/workspace/word-document-from-content-eval/source/brief.md", "bytes": 405, "sha256": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/scope.md", "bytes": 1821, "sha256": "43b5fd7c883e3a46b72239d6ca6609d10e49b48653feadc459d76583ab9dea69", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/sources.md", "bytes": 1648, "sha256": "f6c9cf6e20372f26fe79a7f5549c0048c21e31fec9b8a5192e295ceb24a6409b", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/report.docx", "bytes": 19240, "sha256": "72fc7a22ecc2d24600772db00b1462989ba2895d2e89089cb44d78a37023f7f5", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/report_files/report.md", "bytes": 2142, "sha256": "2211fbd1926ab266e10b16376af3cc046b9e93e3d4794af9b04a75edca2f856b", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:20.966Z", "reviewId": "067", "cellId": "r2-saluki-cuda-productivity-craftbook-research-to-document", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "scenarioId": "craftbook-research-to-document", "status": "scored", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "baselineCellId": "r2-saluki-cuda-productivity-craftbook-research-to-document", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "9ac2e342e26609d01f6dadb0d9f944123e22a0ad4c88d29a1fac128f5c144a25", "evidenceSha256": "5366c9b5e302ddc3fbce24c0d5c010f8cbfba24f8fdee79adb89d4d00a17c0a6", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/workspace/word-document-from-content-eval/documents/eval/report.md", "bytes": 1955, "sha256": "a5b472f4a0c1e1860d5ee3ce070b57063773b94901a14b7178aee84f11c482a1", "shownBytes": 1955, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/workspace/word-document-from-content-eval/source/brief.md", "bytes": 405, "sha256": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "shownBytes": 405, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/scope.md", "bytes": 1821, "sha256": "43b5fd7c883e3a46b72239d6ca6609d10e49b48653feadc459d76583ab9dea69", "shownBytes": 1821, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/sources.md", "bytes": 1648, "sha256": "f6c9cf6e20372f26fe79a7f5549c0048c21e31fec9b8a5192e295ceb24a6409b", "shownBytes": 1648, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/report_files/report.md", "bytes": 2142, "sha256": "2211fbd1926ab266e10b16376af3cc046b9e93e3d4794af9b04a75edca2f856b", "shownBytes": 2142, "truncated": false } ], "contextFiles": [], "scoreAxes": { "grounding": 6, "structure": 9, "convertibility": 8 }, "justification": "All figures (18 SKUs, 18h→6h, 14.2%→8.9%, status silence, three actions) match the brief. But the report adds unsupported claims: status silence is \"the single biggest driver of repeat contact\", the improvement should \"lift customer trust\", and the email design and barcode-stall details are invented.", "limitations": "Source-only review. I did not see the DOCX or any preview, and I did not check the conversion. The scope step's own output was not assessed beyond the saved Markdown. The [S1] inline cites differ slightly from the scope's \"single source note\" plan.", "meanScore": 7.666666666666667, "durationMs": 16203, "costUsd": 0.029374400000000002, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5850, "cache_read_input_tokens": 804, "output_tokens": 589, "output_tokens_details": { "thinking_tokens": 265 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5850, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 589, "cache_read_input_tokens": 804, "cache_creation_input_tokens": 5850, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5850 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 589, "cacheReadInputTokens": 804, "cacheCreationInputTokens": 5850, "webSearchRequests": 0, "costUSD": 0.029374400000000002, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 265, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5850, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-research-to-document / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 25m 50s; budget used: 26.1%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 142136`; `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1951,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2611651100611284`; `facts.toolUse.totalToolCalls: 43`; `facts.timing.timeToFirstArtifactMs: 142136`; `facts.outcome.durationMs: 1549771` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15346 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31409 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.14 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 107033 + 9155 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 8 → 6 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 176s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.63 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 31% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 142136`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1951,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.2611651100611284`\n- `facts.toolUse.totalToolCalls: 43`\n- `facts.timing.timeToFirstArtifactMs: 142136`\n- `facts.outcome.durationMs: 1549771`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 142136`; `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1951,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2611651100611284`; `facts.toolUse.totalToolCalls: 43`; `facts.timing.timeToFirstArtifactMs: 142136`; `facts.outcome.durationMs: 1549771`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory review\n\nFixed result **pass 8.5/10** is unchanged. Declared advisory axes: grounding 4/10; structure 7/10; convertibility 8/10. Mean **6.33/10**, codex-inline/current-session.\n\nAll supplied figures and actions survive in organized Markdown and a real DOCX with a table. Unsupported claims make status silence the biggest repeat-contact driver, say barcode exceptions stall returns, and attribute specific email triggers to the brief. The reviewer explicitly accepts the causal overstatement. Conversion succeeds; source fidelity is weaker.\n\nRe-read the final 1,955-byte Markdown and 405-byte brief. Final source and 19,240-byte DOCX hashes match the previously reviewed snapshot exactly. ZIP/XML and all five recorded reconstructed-import previews were already inspected; they establish content/structure, not Word-native pagination. Read the final review note and terminal step receipt. No new conversion or model run was performed.\n\n## Positives\n\n- All 14 fixed checks pass, all six workflow stages complete, and one real DocBlocks conversion/save preserves the source text and native table. All supplied pilot figures and three next actions are present (outcome.success; sniff.latest.score=14; continuity.steps.completed=6; toolUse.byTool.convert_document=1 and save_artifact=1; advisory-docx-inspection.json).\n- The saved report has a clear title, summary, metrics table, findings and actions. Unlike the first repetition it does not invent an 18-SKU before/after baseline. The threefold response-time ratio follows from 18/6 and is valid derived arithmetic (artifacts.otherFileCount; final report versus source/brief.md).\n\n## Negatives\n\n- The report promotes the top unresolved complaint into the single biggest driver of repeat contact, asserts current barcode-related stalls and expected trust improvement, and presents specific email trigger events as source-backed findings. The brief supplies none of those causal/implementation claims (sniff.latest full pass; artifacts.otherFileCount; final report and brief).\n- At 22:17:00Z the reviewer explicitly notices the repeat-contact claim but calls it an acceptable mild paraphrase because it is not a metric. This demonstrates a source-fidelity review gap, not a missing read or conversion failure (toolUse.byTool.write_task_note=4; advisory-tool-evidence.json).\n- The final review/finish notes cite 19,406 bytes from file-non-empty validation, while the captured binary and converter receipt agree on 19,240 bytes. The reviewer speculates metadata explains this without evidence. No binary change occurred between the provisional and final snapshots; do not report the validator number as actual DOCX file size (toolUse.byTool.validate=1; final artifact SHA-256; advisory-tool-evidence.json).\n- The workflow used 43 calls, including repeated source/artifact reads, so fixed efficiency is 2.5 despite using only 26.1% of its generous budget. Native Word pagination is unverified; the actual preview tool correctly labels its five items reconstructed-import sections (toolUse.totalToolCalls; outcome.budgetUsedFraction; score.axes.efficiency; preview receipts).\n\n## Performance and GPU fit\n\nElapsed **25.83 min**, 43 calls; 9155 generated tokens at **8.14 tok/s**, prefill 357.39 tok/s. Peak VRAM **10646/12227 MiB**; process-tree RSS **15346 MiB**; whole-system RAM **31409/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications. Shared GPU memory and paging were not measured.\n\n1 native CUDA launch, 65,536-token allocation, one slot, f16 KV, reasoning 4096. Dense FFNs for 39/64 layers on GPU and 25 on CPU: hybrid residency, not full GPU loading. No explicit endpoint failure, CUDA OOM or context overflow found in the final log. Maximum captured session fill 31.1%; maximum observed native released-slot size 33178 tokens (50.6%). These are sampled observations, not continuous KV high-water marks. No compaction events observed. Performance is separate from both fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: in Gilde research-to-document craftbook.json based on 1.2.8, steps[id=write].prompt and steps[id=evaluate].prompt, add: “Separate sourced findings from proposed implementations. A causal or superlative claim needs source support even when it contains no number.” Preserve supplied actions; label extra trigger/timing details as proposals. Measure unsupported claims without suppressing useful, clearly labeled recommendations.\n\n## Strategic experiment — not applied\n\nProposed experiment, not applied: packages/service/src/chat/evidence-ledger.ts should preserve source spans for causal claims through handoffs; packages/core/src/checks/claims.ts should distinguish supported paraphrase from a new causal/ranking assertion. Binary validation should surface actual byte length separately from any decoded-text size. Verify the latter discrepancy before changing its implementation.\n\n## Generalization and next experiment\n\nR1 fixed 8.5/advisory 7.33 versus R2 fixed 8.5/advisory 6.33: both publish real DOCX files, but R2 has more unsupported explanations despite better table wording. Compare any provenance experiment with bibliography and Wikipedia briefs, and conversion preservation with the PowerPoint workflow. Same campaign identity; no native-version causal claim.\n\nRun any proposed experiment separately after the frozen campaign. No source, gate, model tuning, model file or runner state was changed. No capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/facts.json): pass, fixed 8.5, 19.63 min, 9.01 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/facts.json): pass, fixed 8.5, 2.56 min, 125.4 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-24-18-896Z-pwh8/facts.json): pass, fixed 8.5, 4.54 min, 119.82 tok/s.\n\nOther-model history is contextual and not a controlled native-version comparison.\n\n## Strategic note\n\nSuccessful conversion and a completed workflow do not establish source-faithful synthesis. The reviewer accepted a known unsupported claim, so additional read calls alone would not address the observed error.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json, historical-comparison.json and final captured files. Additional independent audits: advisory-endpoint-probes.json for Squisq; advisory-key-probes.json for interface-contract; advisory-docx-inspection.json and the preserved provisional snapshot for the Word report.\n" }, { "cellId": "r2-saluki-cuda-developer-fix-squisq-bugs", "baselineCellId": "r2-saluki-cuda-developer-fix-squisq-bugs", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T22:21:46.182Z", "finishedAt": "2026-11-05T22:30:28.745Z", "durationMs": 522563, "success": true, "reason": "all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 316532 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "fix-squisq-bugs", "bytes": 6171, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.15851079093263784 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 316532 }, { "source": "facts", "path": "outcome.durationMs", "value": 522563 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 87, "peakRssMb": 9939, "peakCpuPercent": 1197.7, "firstAtMs": 1, "lastAtMs": 508766 }, "gpu": { "available": true, "sampleCount": 87, "peakUtilPercent": 100, "peakMemUsedMb": 10624, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 87, "peakUsedMb": 27292, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 8135, "totalOutputTokens": 3919, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T22:30:23.708Z" } }, "source": "engine-log" }, "derived": { "durationMs": 511278, "meanTokensPerSec": 8.24, "genTokensPerSec": 8.24, "genTokens": 3919, "promptTokensPerSec": 535.4, "peakRssMb": 9939, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T22:21:51.919Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T22:21:51.824Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "dani": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0513763427734375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0513763427734375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)", "durationMs": 522563, "timeoutMs": 3296703, "budgetUsedFraction": 0.15851079093263784 }, "timing": { "startedAt": "2026-11-05T22:21:46.182Z", "finishedAt": "2026-11-05T22:30:28.745Z", "timeToFirstArtifactMs": 316532, "timeToLastArtifactWriteMs": 510677, "timeToFirstTokenMs": 24500, "firstTurnTtftMs": 13960, "timeToFirstToolCallMs": 40885 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "read_file": 4, "replace_in_file": 5 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 7846, "filePath": "fix-squisq-bugs", "bytes": 5461, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng 87.8 — it must cross the antimeridian (±180), not run back through 0" }, { "atMs": 319614, "filePath": "fix-squisq-bugs", "bytes": 5927, "score": 2, "scoreMax": 5, "signals": [ "great-circle-path", "haversine-untouched" ], "failReason": "graceful-errors behavioral check failed: getGeohash4Neighbors expects a 4-char geohash, got \"xx\"" }, { "atMs": 516989, "filePath": "fix-squisq-bugs", "bytes": 6171, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } ], "latest": { "filePath": "fix-squisq-bugs", "bytes": 6171, "score": 5, "scoreMax": 5, "signals": [ "great-circle-path", "graceful-errors", "deep-layer-validation", "haversine-untouched", "tsc-clean" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T22:21:46.185Z] [trial] id=fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3 scenario=fix-squisq-bugs model=underdog-saluki-27b-iq2-mix", "[2026-11-05T22:21:46.185Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "[2026-11-05T22:21:46.185Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T22:21:46.185Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-fix-squisq-bugs-o3j15O", "[2026-11-05T22:21:46.185Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T22:21:46.188Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T22:21:46.193Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T22:21:46.199Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-fix-squisq-bugs-o3j15O", "[2026-11-05T22:21:46.199Z] [trial] throughput-scaled ceiling: 25m → 55m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T22:21:51.877Z] [trial] daemon spawned pid=15900 port=https://127.0.0.1:59520", "[2026-11-05T22:21:53.297Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T22:21:53.314Z] [trial] running scenario setup hook", "[2026-11-05T22:21:53.314Z] [trial] meester=torsten", "[2026-11-05T22:21:53.748Z] [trial] scenario setup complete", "[2026-11-05T22:21:53.750Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T22:21:53.818Z] [poll] initial digests hard=b97ba050b5346f64 soft=4a2adf9a9d172e7b", "[2026-11-05T22:21:54.129Z] [poll] hard-progress: b97ba050b5346f64 → 7a455b7b7e97a692 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/5461B)", "[2026-11-05T22:22:28.634Z] [poll] hard-progress: 7a455b7b7e97a692 → ef1729529cf80c6b after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=37,stream=107) sniff=1/5461B)", "[2026-11-05T22:27:05.902Z] [poll] hard-progress: ef1729529cf80c6b → 9ef6d5b330d38af0 after 277s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=759,stream=2887) sniff=2/5927B)", "[2026-11-05T22:30:23.171Z] [poll] terminal: success=true reason=all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)", "[2026-11-05T22:30:23.760Z] [capture] dumped 1 session(s)", "[2026-11-05T22:30:23.764Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-05T22:30:23.862Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T22:30:28.743Z] [trial] daemon shut down", "[2026-11-05T22:30:28.753Z] [trial] PASS duration=522563ms reason=all 5 signals firing including tsc-clean (great-circle-path, graceful-errors, deep-layer-validation, haversine-untouched, tsc-clean)" ] }, "metrics": { "process": { "sampleCount": 87, "peakRssMb": 9939, "peakCpuPercent": 1197.7, "firstAtMs": 1, "lastAtMs": 508766 }, "gpu": { "available": true, "sampleCount": 87, "peakUtilPercent": 100, "peakMemUsedMb": 10624, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 87, "peakUsedMb": 27292, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 8135, "totalOutputTokens": 3919, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T22:30:23.708Z" } }, "source": "engine-log" }, "derived": { "durationMs": 511278, "meanTokensPerSec": 8.24, "genTokensPerSec": 8.24, "genTokens": 3919, "promptTokensPerSec": 535.4, "peakRssMb": 9939, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T22:21:46.182Z", "finishedAt": "2026-11-05T22:30:28.745Z", "timeToFirstArtifactMs": 316532, "timeToLastArtifactWriteMs": 510677, "timeToFirstTokenMs": 24500, "firstTurnTtftMs": 13960, "timeToFirstToolCallMs": 40885 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T22:21:51.919Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T22:21:51.824Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "dani": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0513763427734375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0513763427734375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-fix-squisq-bugs", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "scenarioId": "fix-squisq-bugs", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-05T23:08:01.382Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "artifactBytes": 3944, "artifactSha256": "cdca05d472bdd3b534e30691244fbdefb8501a4202b8049bac43530f42397643", "axisSource": null, "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared qualitative axes. All five fixed checks pass, and invalid-input returns plus layer validation work. Independent read-only probes show the revised great-circle function still misses destinations by 33–3,258 km on diverse routes and emits longitudes beyond 180 degrees. Compilation and the narrow antimeridian gate do not establish geodesic correctness.", "reviewBasis": "Read both changed TypeScript files, Haversine.ts, the seeded bug report and standard behavioral gate. Transpiled only captured pure functions in an isolated VM and checked endpoints for four diverse routes plus a coincident case, malformed layers and invalid-length returns. Results are saved separately in advisory-endpoint-probes.json; fixed scores and model artifacts are unchanged.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "bytes": 3944, "sha256": "cdca05d472bdd3b534e30691244fbdefb8501a4202b8049bac43530f42397643", "status": "reviewedNoDeclaredAxes" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/imageEdit/persistence.ts", "bytes": 2227, "sha256": "15d7ab106f1e475991d992ec252800399a9d5539704ec1e54caaf7c7ab48b3bd", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/spatial/Haversine.ts", "bytes": 749, "sha256": "6ba31700e271f0f3a33aff0339cb30e902bb890c78d74bb9c47f2e4b06eac635", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/bug_report.md", "bytes": 1324, "sha256": "d44d282baa62957db8f6d72411f89a68d8e5fbd11ec4fa1721a87f779def6a2b", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:19.015Z", "reviewId": "068", "cellId": "r2-saluki-cuda-developer-fix-squisq-bugs", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "scenarioId": "fix-squisq-bugs", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "baselineCellId": "r2-saluki-cuda-developer-fix-squisq-bugs", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "491583ba9d88d395b1e92a1c3c0fccd4852ea9a5c6964eab78f02c087356fce2", "evidenceSha256": "cead075fae2e203870206711722ff69902848e69d0a3cd7f94f8d89fa6e38250", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "bytes": 3944, "sha256": "cdca05d472bdd3b534e30691244fbdefb8501a4202b8049bac43530f42397643", "shownBytes": 3944, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/imageEdit/persistence.ts", "bytes": 2227, "sha256": "15d7ab106f1e475991d992ec252800399a9d5539704ec1e54caaf7c7ab48b3bd", "shownBytes": 2227, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/spatial/Haversine.ts", "bytes": 749, "sha256": "6ba31700e271f0f3a33aff0339cb30e902bb890c78d74bb9c47f2e4b06eac635", "shownBytes": 749, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/bug_report.md", "bytes": 1324, "sha256": "d44d282baa62957db8f6d72411f89a68d8e5fbd11ec4fa1721a87f779def6a2b", "shownBytes": 1324, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "All three bugs appear fixed. getGeohashPath uses haversine distance for step count with great-circle (bearing/destination-point) interpolation. assertImageEditDoc validates every layer's id, type and finite x/y. Both neighbor and hierarchical functions return []/'' on bad length. Haversine.ts is unchanged.", "limitations": "Source-only review. No tsc or runtime check. Antimeridian and degenerate cases (same-cell path, antipodal points) were not exercised. Whether Haversine.ts was edited cannot be verified without a diff, though its content looks intact. Transcript not shown.", "meanScore": null, "durationMs": 14164, "costUsd": 0.027324, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6005, "cache_read_input_tokens": 700, "output_tokens": 323, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6005, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 323, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 6005, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6005 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 323, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 6005, "webSearchRequests": 0, "costUSD": 0.027324, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6005, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - fix-squisq-bugs / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 8m 43s; budget used: 15.9%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 316532`; `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":6171,\"score\":5,\"scoreMax\":5,\"signals\":[\"great-circle-path\",\"graceful-errors\",\"deep-layer-validation\",\"haversine-untouched\",\"tsc-clean\"],\"fa...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.15851079093263784`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 316532`; `facts.outcome.durationMs: 522563` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9939 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10624 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27292 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.24 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 8135 + 3919 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 5% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 316532`\n- `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":6171,\"score\":5,\"scoreMax\":5,\"signals\":[\"great-circle-path\",\"graceful-errors\",\"deep-layer-validation\",\"haversine-untouched\",\"tsc-clean\"],\"fa...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.15851079093263784`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 316532`\n- `facts.outcome.durationMs: 522563`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 316532`; `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":6171,\"score\":5,\"scoreMax\":5,\"signals\":[\"great-circle-path\",\"graceful-errors\",\"deep-layer-validation\",\"haversine-untouched\",\"tsc-clean\"],\"fa...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.15851079093263784`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 316532`; `facts.outcome.durationMs: 522563`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory review\n\nFixed result **pass 10/10** is unchanged. Advisory status **notApplicable**; no axis scores invented.\n\nNo declared qualitative axes. All five fixed checks pass, and invalid-input returns plus layer validation work. Independent read-only probes show the revised great-circle function still misses destinations by 33–3,258 km on diverse routes and emits longitudes beyond 180 degrees. Compilation and the narrow antimeridian gate do not establish geodesic correctness.\n\nRead both changed TypeScript files, Haversine.ts, the seeded bug report and standard behavioral gate. Transpiled only captured pure functions in an isolated VM and checked endpoints for four diverse routes plus a coincident case, malformed layers and invalid-length returns. Results are saved separately in advisory-endpoint-probes.json; fixed scores and model artifacts are unchanged.\n\n## Positives\n\n- All five fixed signals pass, including clean TypeScript and unchanged Haversine.ts. The model completes nine tools with no red flags or automatic answers (outcome.success; sniff.latest.signals; toolUse.totalToolCalls=9; toolUse.redFlags; autoAnswer.total).\n- Invalid geohash lengths return [] and an empty string as requested. Per-layer checks reject null, unsupported types and nonfinite coordinates while accepting a valid layer (toolUse.byTool.replace_in_file=5; sniff.latest graceful-errors/deep-layer-validation; advisory-endpoint-probes.json).\n\n## Negatives\n\n- The new formula omits cos(lat2) from the initial-bearing numerator. Its longitude calculation also omits cos(lat1) from the numerator and introduces an extra cos(bearing) in the denominator. These are substantive formula errors despite the fixed great-circle-path signal (sniff.latest.signals; final Geohash.ts; advisory-endpoint-probes.json).\n- Read-only endpoint probes show approximately 32.62 km error for (10,170)→(10,-170), 1,593.86 km for New York→London, 493.16 km for Sydney→Tokyo and 3,257.87 km for a 70-degree-latitude route. The antimeridian case ends at longitude 189.705 degrees instead of normalized -169.997; the coincident case succeeds (sniff.latest full pass; artifacts.otherFileCount; advisory-endpoint-probes.json).\n- No model-side test execution appears among the nine calls: four read_file and five replace_in_file. The standard gate compiles and performs a narrow equatorial route probe, but does not establish arbitrary-route endpoint accuracy (toolUse.byTool; outcome.reason; evals/src/scenarios/fix-squisq-bugs.ts behavioral check).\n\n## Performance and GPU fit\n\nElapsed **8.71 min**, 9 calls; 3919 generated tokens at **8.24 tok/s**, prefill 535.4 tok/s. Peak VRAM **10624/12227 MiB**; process-tree RSS **9939 MiB**; whole-system RAM **27292/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications. Shared GPU memory and paging were not measured.\n\n1 native CUDA launch, 65,536-token allocation, one slot, f16 KV, reasoning 4096. Dense FFNs for 39/64 layers on GPU and 25 on CPU: hybrid residency, not full GPU loading. No explicit endpoint failure, CUDA OOM or context overflow found in the final log. Maximum captured session fill 5.1%; maximum observed native released-slot size 9682 tokens (14.8%). These are sampled observations, not continuous KV high-water marks. No compaction events observed. Performance is separate from both fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: add packages/core/src/model-profile/behaviors/numeric-verification.ts with a reusable prompt: “Before declaring a numeric algorithm repaired, verify one boundary identity and one nontrivial independent case; include an execution receipt or mark verification pending.” Do not insert named cities, this fixture or a ready-made formula into model tuning. Measure semantic errors and verification cost before enabling the behavior.\n\n## Strategic experiment — not applied\n\nProposed experiment, not applied: extend the existing behavioral check in evals/src/scenarios/fix-squisq-bugs.ts with endpoint, longitude-range and route-reversal invariants across generated coordinate pairs. Pair this evaluation correction with a reusable isolated function-probe tool rather than teaching the model the fixed expected answers. Fixed comparison gates remain frozen for this campaign.\n\n## Generalization and next experiment\n\nR1 fixed 9.5 and R2 fixed 10 both leave geodesic math wrong, using different incorrect formulas. R2 is faster and has no native recovery event, but the higher composite does not show a corrected path algorithm. Validate numeric-invariant tooling against the unrelated A/B statistics and spreadsheet calculations; measure semantic errors independently of gate pass counts.\n\nRun any proposed experiment separately after the frozen campaign. No source, gate, model tuning, model file or runner state was changed. No capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/facts.json): pass, fixed 9.5, 14.38 min, 8.66 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/facts.json): pass, fixed 8.5, 7.59 min, 132.02 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-34-13-154Z-i0ht/facts.json): pass, fixed 9.5, 1.14 min, 139.43 tok/s.\n\nOther-model history is contextual and not a controlled native-version comparison.\n\n## Strategic note\n\nThis is a recurring semantic-validation gap under successful compilation and narrow behavioral acceptance. Neither tuning nor the native build changed between repetitions. Preserve the incorrect output as evidence rather than retrying it.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json, historical-comparison.json and final captured files. Additional independent audits: advisory-endpoint-probes.json for Squisq; advisory-key-probes.json for interface-contract; advisory-docx-inspection.json and the preserved provisional snapshot for the Word report.\n" }, { "cellId": "r2-saluki-cuda-developer-dev-craftbook-routing", "baselineCellId": "r2-saluki-cuda-developer-dev-craftbook-routing", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T22:32:46.674Z", "finishedAt": "2026-11-05T22:34:20.397Z", "durationMs": 93723, "success": true, "reason": "task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.02842931255863813 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 1 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 93723 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" }, "facts": { "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 15, "peakRssMb": 9762, "peakCpuPercent": 1196.9, "firstAtMs": 0, "lastAtMs": 82608 }, "gpu": { "available": true, "sampleCount": 15, "peakUtilPercent": 96, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 27341, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11816, "totalOutputTokens": 386, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T22:34:16.165Z" } }, "source": "engine-log" }, "derived": { "durationMs": 83791, "meanTokensPerSec": 9.09, "genTokensPerSec": 9.09, "genTokens": 386, "promptTokensPerSec": 640.35, "peakRssMb": 9762, "peakGpuUtilPercent": 96, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "batch-exporter/1", "stepId": "reproduce", "activatedAt": "2026-11-05T22:34:13.652Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T22:32:52.355Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T22:32:52.263Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "imara": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.153533935546875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.153533935546875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "durationMs": 93723, "timeoutMs": 3296703, "budgetUsedFraction": 0.02842931255863813 }, "timing": { "startedAt": "2026-11-05T22:32:46.674Z", "finishedAt": "2026-11-05T22:34:20.397Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 43391, "firstTurnTtftMs": 26005, "timeToFirstToolCallMs": 87146 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 1, "byTool": { "start_work": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 3 }, "sniff": { "progression": [ { "atMs": 7097, "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } ], "latest": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T22:32:46.678Z] [trial] id=dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm scenario=dev-craftbook-routing model=underdog-saluki-27b-iq2-mix", "[2026-11-05T22:32:46.679Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "[2026-11-05T22:32:46.679Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-dev-craftbook-routing-f6Fgdo", "[2026-11-05T22:32:46.679Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T22:32:46.679Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T22:32:46.681Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T22:32:46.686Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T22:32:46.692Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-dev-craftbook-routing-f6Fgdo", "[2026-11-05T22:32:46.692Z] [trial] throughput-scaled ceiling: 25m → 55m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T22:32:52.318Z] [trial] daemon spawned pid=28140 port=https://127.0.0.1:59255", "[2026-11-05T22:32:53.177Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T22:32:53.194Z] [trial] meester=imara", "[2026-11-05T22:32:53.194Z] [trial] running scenario setup hook", "[2026-11-05T22:32:53.484Z] [trial] scenario setup complete", "[2026-11-05T22:32:53.486Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T22:32:53.558Z] [poll] initial digests hard=a74f6fd14cf8e0f3 soft=15a4d2ca170923a9", "[2026-11-05T22:32:53.832Z] [poll] hard-progress: a74f6fd14cf8e0f3 → f4718e4dd223619f after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/3B)", "[2026-11-05T22:34:15.961Z] [poll] terminal: success=true reason=task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "[2026-11-05T22:34:16.191Z] [capture] dumped 1 session(s)", "[2026-11-05T22:34:16.195Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-05T22:34:16.267Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-05T22:34:20.395Z] [trial] daemon shut down", "[2026-11-05T22:34:20.404Z] [trial] PASS duration=93723ms reason=task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored" ] }, "metrics": { "process": { "sampleCount": 15, "peakRssMb": 9762, "peakCpuPercent": 1196.9, "firstAtMs": 0, "lastAtMs": 82608 }, "gpu": { "available": true, "sampleCount": 15, "peakUtilPercent": 96, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 15, "peakUsedMb": 27341, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11816, "totalOutputTokens": 386, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T22:34:16.165Z" } }, "source": "engine-log" }, "derived": { "durationMs": 83791, "meanTokensPerSec": 9.09, "genTokensPerSec": 9.09, "genTokens": 386, "promptTokensPerSec": 640.35, "peakRssMb": 9762, "peakGpuUtilPercent": 96, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T22:32:46.674Z", "finishedAt": "2026-11-05T22:34:20.397Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 43391, "firstTurnTtftMs": 26005, "timeToFirstToolCallMs": 87146 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "batch-exporter/1", "stepId": "reproduce", "activatedAt": "2026-11-05T22:34:13.652Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T22:32:52.355Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T22:32:52.263Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "imara": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.153533935546875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.153533935546875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-dev-craftbook-routing", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "scenarioId": "dev-craftbook-routing", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-05T23:08:01.431Z", "artifactPath": null, "artifactBytes": 0, "artifactSha256": null, "axisSource": null, "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "Selection-only success: start_work chose the bundled bug-fix-tdd recipe and dispatched its reproduce step. No authored deliverable was required or produced; seeded files are not generated artifacts. This establishes routing, not execution or completion of the requested code fix.", "reviewBasis": "Read the single start_work receipt, scenario result, frozen facts/score and captured file inventory. The task was started with explicit regression-test-first instructions. No qualitative artifact axes apply, and no score is invented for the seeded code.", "deliverables": [] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:20.119Z", "reviewId": "069", "cellId": "r2-saluki-cuda-developer-dev-craftbook-routing", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "scenarioId": "dev-craftbook-routing", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "baselineCellId": "r2-saluki-cuda-developer-dev-craftbook-routing", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "d5b38c926ffea4375ac4e1042bc9e6de370f15252746689836ca2fbc569bcaf3", "evidenceSha256": "90ea23b81b980cb8ed0e33456f57b7601a1eabc05af20bc0d1f7b6908a3ba268", "artifactFiles": [], "contextFiles": [], "scoreAxes": {}, "justification": "The deliverable was absent at closeout: artifacts is empty, no declared axes, no missing paths listed, and no context files. The evidence has no transcript, tool calls, or workspace state showing whether a task was started from a recipe, so the cause of the missing output can't be determined.", "limitations": "Disposition is runner-terminal, but no runner outcome or partial work is shown. I can't tell whether the agent chose a recipe, started a task, or stopped early. No scores are assigned, and this is not a judgment of model success or failure.", "meanScore": null, "durationMs": 13911, "costUsd": 0.009076, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1523, "cache_read_input_tokens": 700, "output_tokens": 291, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1523, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 291, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1523, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1523 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 291, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1523, "webSearchRequests": 0, "costUSD": 0.009076, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1523, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - dev-craftbook-routing / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.8 / 10** (band: ship-ready)\n**Outcome: success** (duration: 1m 34s; budget used: 2.8%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 5 / 10 | The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.02842931255863813`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 93723` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **8.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9762 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 96% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27341 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.09 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 11816 + 386 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 50% of its measured signals.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.02842931255863813`\n- `facts.toolUse.totalToolCalls: 1`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 93723`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.02842931255863813`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 93723`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory review\n\nFixed result **pass 8.8/10** is unchanged. Advisory status **noArtifact**; no axis scores invented.\n\nSelection-only success: start_work chose the bundled bug-fix-tdd recipe and dispatched its reproduce step. No authored deliverable was required or produced; seeded files are not generated artifacts. This establishes routing, not execution or completion of the requested code fix.\n\nRead the single start_work receipt, scenario result, frozen facts/score and captured file inventory. The task was started with explicit regression-test-first instructions. No qualitative artifact axes apply, and no score is invented for the seeded code.\n\n## Positives\n\n- The single start_work call selects craftbook:bug-fix-tdd, creates batch-exporter/1 with eight steps and dispatches reproduce. The details preserve the requested failing-test-before-fix order (outcome.success; outcome.reason; toolUse.totalToolCalls=1; advisory-tool-evidence.json).\n- No new craftbook is authored, and no tool red flag or automatic answer appears (outcome.reason; toolUse.redFlags; autoAnswer.total).\n\n## Negatives\n\n- The probe ends at selection, so it supplies no evidence of a completed regression test or fixed batching behavior. Only seeded workspace code exists and no authored deliverable is judged (outcome.reason; toolUse.byTool.start_work=1; artifacts.otherFileCount; captured inventory).\n- The last sniff still describes a missing recipe task even though the terminal result records successful selection. The fixed 8.8 composite includes a 5/10 output-quality contribution from that stale sniff; do not present the composite as a failed routing check (sniff.latest.score=1/2; sniff.latest.failReason; outcome.success; score.axes.quality).\n\n## Performance and GPU fit\n\nElapsed **1.56 min**, 1 calls; 386 generated tokens at **9.09 tok/s**, prefill 640.35 tok/s. Peak VRAM **10628/12227 MiB**; process-tree RSS **9762 MiB**; whole-system RAM **27341/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications. Shared GPU memory and paging were not measured.\n\n1 native CUDA launch, 65,536-token allocation, one slot, f16 KV, reasoning 4096. Dense FFNs for 39/64 layers on GPU and 25 on CPU: hybrid residency, not full GPU loading. No explicit endpoint failure, CUDA OOM or context overflow found in the final log. Maximum captured session fill 15.4%; maximum observed native released-slot size 12201 tokens (18.6%). These are sampled observations, not continuous KV high-water marks. No compaction events observed. Performance is separate from both fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: retain the current recipe choice and test-first intent; in packages/service/src/chat/manager.ts work-start response handling, derive the user-facing noun from the selected craftbook output type. The start_work receipt here says “your document is underway” for a code repair. Measure response accuracy across coding and document requests without changing selection policy.\n\n## Strategic experiment — not applied\n\nProposed experiment, not applied: in evals/src/score-trial.ts distinguish selection-only acceptance from deliverable quality and reconcile the terminal selection result with the captured sniff. Keep routing and downstream execution as separate evidence dimensions rather than assigning a task-completion interpretation to a route probe.\n\n## Generalization and next experiment\n\nR1 and R2 both pass at fixed 8.8 with one start_work call and no authored artifact. Compare with the same campaign bug-fix-tdd execution trial before drawing conclusions about completed developer work; compare response wording against research-to-document to keep the proposal general.\n\nRun any proposed experiment separately after the frozen campaign. No source, gate, model tuning, model file or runner state was changed. No capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/facts.json): pass, fixed 8.8, 1.28 min, 9.23 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/facts.json): pass, fixed 8.8, 1.21 min, 110.02 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-47-38-234Z-ed7n/facts.json): fail, fixed 2, 5.41 min, 112 tok/s.\n\nOther-model history is contextual and not a controlled native-version comparison.\n\n## Strategic note\n\nRouting is stable across repetitions. A successful route is useful evidence, but it must not be counted as a completed bug fix in the final diverse-task assessment.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json, historical-comparison.json and final captured files. Additional independent audits: advisory-endpoint-probes.json for Squisq; advisory-key-probes.json for interface-contract; advisory-docx-inspection.json and the preserved provisional snapshot for the Word report.\n" }, { "cellId": "r2-saluki-cuda-developer-interface-contract", "baselineCellId": "r2-saluki-cuda-developer-interface-contract", "suite": "developer", "scenario": "interface-contract", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T22:36:51.369Z", "finishedAt": "2026-11-05T22:42:39.846Z", "durationMs": 348477, "success": true, "reason": "tsc emit clean + node integration.mjs passed: 5 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 255517 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "interface-contract", "bytes": 1443, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08808724068791955 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 7 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 255517 }, { "source": "facts", "path": "outcome.durationMs", "value": 348477 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 58, "peakRssMb": 11863, "peakCpuPercent": 1231.1, "firstAtMs": 0, "lastAtMs": 335988 }, "gpu": { "available": true, "sampleCount": 58, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 58, "peakUsedMb": 28577, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 15902, "totalOutputTokens": 2364, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15291, "todayTokensCached": 14134, "todayTokensOut": 626, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15291, "totalTokensCached": 14134, "totalTokensOut": 626, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7732, "cachedInputTokens": 7067, "outputTokens": 484, "cost": 0, "durationMs": 58594, "outputTokensPerSec": 8.699883156786646, "at": "2026-11-05T22:39:18.749Z" }, "lastUpdated": "2026-11-05T22:39:18.749Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T22:42:35.330Z" } }, "source": "engine-log" }, "derived": { "durationMs": 338300, "meanTokensPerSec": 8.82, "genTokensPerSec": 8.82, "genTokens": 2364, "promptTokensPerSec": 424.35, "peakRssMb": 11863, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15291, "output": 626 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "event-pipeline/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-05T22:38:02.766Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "e114ac85-7720-4e6f-8a18-da6b2e502ff7" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T22:36:57.014Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T22:36:56.912Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "esperanza": 2, "vanya": 1 }, "perTask": { "event-pipeline/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.10809326171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.10809326171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tsc emit clean + node integration.mjs passed: 5 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "durationMs": 348477, "timeoutMs": 3956044, "budgetUsedFraction": 0.08808724068791955 }, "timing": { "startedAt": "2026-11-05T22:36:51.369Z", "finishedAt": "2026-11-05T22:42:39.846Z", "timeToFirstArtifactMs": 255517, "timeToLastArtifactWriteMs": 337141, "timeToFirstTokenMs": 27911, "firstTurnTtftMs": 18634, "timeToFirstToolCallMs": 71457 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 7, "byTool": { "start_work": 1, "search_history": 1, "message_gezel": 1, "list_dir": 1, "write_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 343568, "filePath": "interface-contract", "bytes": 1443, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } ], "latest": { "filePath": "interface-contract", "bytes": 1443, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T22:36:51.375Z] [trial] id=interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y scenario=interface-contract model=underdog-saluki-27b-iq2-mix", "[2026-11-05T22:36:51.375Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "[2026-11-05T22:36:51.375Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T22:36:51.375Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T22:36:51.375Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-interface-contract-gWhPeR", "[2026-11-05T22:36:51.377Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T22:36:51.382Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T22:36:51.386Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-interface-contract-gWhPeR", "[2026-11-05T22:36:51.386Z] [trial] throughput-scaled ceiling: 30m → 66m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T22:36:56.989Z] [trial] daemon spawned pid=8944 port=https://127.0.0.1:57227", "[2026-11-05T22:36:57.832Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T22:36:57.848Z] [trial] meester=vanya", "[2026-11-05T22:36:57.991Z] [trial] sent prompt to meester (1435 chars)", "[2026-11-05T22:36:58.045Z] [poll] initial digests hard=d4823e3cc875f31d soft=29fbed2d9729d3a2", "[2026-11-05T22:38:02.559Z] [poll] hard-progress: d4823e3cc875f31d → 5464dd19f0015e1c after 65s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=0,slot=130,stream=666) sniff=0/0B)", "[2026-11-05T22:38:08.508Z] [poll] hard-progress: 5464dd19f0015e1c → 7a949895b69d91cd after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=142,stream=701) sniff=0/0B)", "[2026-11-05T22:39:07.394Z] [poll] hard-progress: 7a949895b69d91cd → 23806a8269005d13 after 59s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=1,slot=307,stream=1185) sniff=0/0B)", "[2026-11-05T22:39:49.097Z] [poll] hard-progress: 23806a8269005d13 → 0c09c15b4f41d926 after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=362,stream=1325) sniff=0/0B)", "[2026-11-05T22:40:47.908Z] [poll] hard-progress: 0c09c15b4f41d926 → f6317b1175b14554 after 59s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=3,slot=522,stream=1799) sniff=0/0B)", "[2026-11-05T22:41:11.640Z] [poll] hard-progress: f6317b1175b14554 → 7e4e6d7b146c2c80 after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=568,stream=1993) sniff=1/0B)", "[2026-11-05T22:41:29.338Z] [poll] hard-progress: 7e4e6d7b146c2c80 → efd20676ac0d90a6 after 18s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=5,slot=609,stream=2115) sniff=1/0B)", "[2026-11-05T22:42:34.943Z] [poll] terminal: success=true reason=tsc emit clean + node integration.mjs passed: 5 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "[2026-11-05T22:42:35.393Z] [capture] dumped 3 session(s)", "[2026-11-05T22:42:35.398Z] [capture] snapshotted telemetry for 2 session(s)", "[2026-11-05T22:42:35.467Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)" ] }, "metrics": { "process": { "sampleCount": 58, "peakRssMb": 11863, "peakCpuPercent": 1231.1, "firstAtMs": 0, "lastAtMs": 335988 }, "gpu": { "available": true, "sampleCount": 58, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 58, "peakUsedMb": 28577, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 15902, "totalOutputTokens": 2364, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15291, "todayTokensCached": 14134, "todayTokensOut": 626, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15291, "totalTokensCached": 14134, "totalTokensOut": 626, "totalCost": 0, "medianOutputTokensPerSec": 8.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.9, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7732, "cachedInputTokens": 7067, "outputTokens": 484, "cost": 0, "durationMs": 58594, "outputTokensPerSec": 8.699883156786646, "at": "2026-11-05T22:39:18.749Z" }, "lastUpdated": "2026-11-05T22:39:18.749Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T22:42:35.330Z" } }, "source": "engine-log" }, "derived": { "durationMs": 338300, "meanTokensPerSec": 8.82, "genTokensPerSec": 8.82, "genTokens": 2364, "promptTokensPerSec": 424.35, "peakRssMb": 11863, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15291, "output": 626 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T22:36:51.369Z", "finishedAt": "2026-11-05T22:42:39.846Z", "timeToFirstArtifactMs": 255517, "timeToLastArtifactWriteMs": 337141, "timeToFirstTokenMs": 27911, "firstTurnTtftMs": 18634, "timeToFirstToolCallMs": 71457 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "event-pipeline/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-05T22:38:02.766Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "e114ac85-7720-4e6f-8a18-da6b2e502ff7" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T22:36:57.014Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T22:36:56.912Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "esperanza": 2, "vanya": 1 }, "perTask": { "event-pipeline/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.10809326171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.10809326171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-interface-contract", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "scenarioId": "interface-contract", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-05T23:08:01.479Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/types.ts", "artifactBytes": 330, "artifactSha256": "90aef81b66c3f49e583a00a53971b4ad0547a74523010eb128fcaa94ba7ceb23", "axisSource": null, "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared qualitative axes. The three files compile and route five events across four kinds, but a normal-object accumulator corrupts counts for constructor, toString and __proto__. All events still reach the sink. One worker wrote the code, so independent multi-worker contract coordination was not exercised.", "reviewBasis": "Read the complete three-file implementation and original string-kind contract. The normal integration gate passed. Read-only probes of the captured consumer after TypeScript transpilation verify the inherited-key failure; results are in advisory-key-probes.json. No model rerun, source repair or fixed-score change.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/types.ts", "bytes": 330, "sha256": "90aef81b66c3f49e583a00a53971b4ad0547a74523010eb128fcaa94ba7ceb23", "status": "reviewedNoDeclaredAxes" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/producer.ts", "bytes": 656, "sha256": "dc3d0a72617621a1d3ea340669fe36734ef18e6758c1b6793a693355e157e00e", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/consumer.ts", "bytes": 457, "sha256": "c63ebe35d14a9995ca104c220af08cfbfdfb9de1d6a97217f731ea9836d008bc", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:33.646Z", "reviewId": "070", "cellId": "r2-saluki-cuda-developer-interface-contract", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "scenarioId": "interface-contract", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "baselineCellId": "r2-saluki-cuda-developer-interface-contract", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "127e8d9326e6a325cd641dff9fe52502e28f3748eef6972ef9147f82314d17bc", "evidenceSha256": "087cab241d6d161d0e1ee548cc6211675b0ff601339ca7737d92277a31e533d6", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/types.ts", "bytes": 330, "sha256": "90aef81b66c3f49e583a00a53971b4ad0547a74523010eb128fcaa94ba7ceb23", "shownBytes": 330, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/producer.ts", "bytes": 656, "sha256": "dc3d0a72617621a1d3ea340669fe36734ef18e6758c1b6793a693355e157e00e", "shownBytes": 656, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/consumer.ts", "bytes": 457, "sha256": "c63ebe35d14a9995ca104c220af08cfbfdfb9de1d6a97217f731ea9836d008bc", "shownBytes": 457, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "Source looks correct. types.ts defines the shared EventRecord and Sink. producer.ts returns 5 events across 4 kinds, each with a payload. consumer.ts routes every event to sink.handle and returns counts by kind. Both import from \"./types.ts\" with explicit extensions. No dependencies. No numeric axes were declared.", "limitations": "The evidence is source-only. It doesn't show tsc or integration.mjs running. It doesn't show the project being created or the work being split between developers. Without allowImportingTsExtensions configured, tsc may reject the .ts imports, and no tsconfig is shown. Non-type imports lack the `type` keyword, which could matter under some settings.", "meanScore": null, "durationMs": 14579, "costUsd": 0.01444, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2724, "cache_read_input_tokens": 700, "output_tokens": 347, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2724, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 347, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2724, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2724 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 347, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2724, "webSearchRequests": 0, "costUSD": 0.01444, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2724, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - interface-contract / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 5m 48s; budget used: 8.8%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 255517`; `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1443,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.08808724068791955`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 255517`; `facts.outcome.durationMs: 348477` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11863 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28577 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.82 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 15902 + 2364 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 11% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 255517`\n- `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1443,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.08808724068791955`\n- `facts.toolUse.totalToolCalls: 7`\n- `facts.timing.timeToFirstArtifactMs: 255517`\n- `facts.outcome.durationMs: 348477`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 255517`; `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1443,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.08808724068791955`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 255517`; `facts.outcome.durationMs: 348477`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory review\n\nFixed result **pass 10/10** is unchanged. Advisory status **notApplicable**; no axis scores invented.\n\nNo declared qualitative axes. The three files compile and route five events across four kinds, but a normal-object accumulator corrupts counts for constructor, toString and __proto__. All events still reach the sink. One worker wrote the code, so independent multi-worker contract coordination was not exercised.\n\nRead the complete three-file implementation and original string-kind contract. The normal integration gate passed. Read-only probes of the captured consumer after TypeScript transpilation verify the inherited-key failure; results are in advisory-key-probes.json. No model rerun, source repair or fixed-score change.\n\n## Positives\n\n- The gate compiles all three files and confirms every one of five events across four distinct kinds reaches sink.handle with the original payload and expected ordinary counts. Both implementation modules import the shared contract (outcome.success; outcome.reason; sniff.latest.signals; final source).\n- Seven calls and 8.8% budget use yield fixed efficiency 10, with no red flags or automatic answers. The code is concise and uses unknown payloads rather than weakening the shared interface to any (toolUse.totalToolCalls; outcome.budgetUsedFraction; toolUse.redFlags; src/types.ts).\n\n## Negatives\n\n- counts is initialized as {} and reads inherited properties before incrementing. Two constructor or toString events produce function-text strings; __proto__ produces no own numeric count. Routing still occurs twice in each probe, so the defect is in aggregation (sniff.latest full pass; artifacts.otherFileCount; advisory-key-probes.json).\n- The producer comment says three kinds but supplies four. This is a minor stale comment; the substantive contract issue is arbitrary string keys (outcome.reason four kinds; captured producer.ts).\n- The terminal result reports one distinct non-Meester worker session. This demonstrates a solo shared-type implementation, not separately produced modules successfully coordinated across developers (outcome.reason; continuity.sessions.byGezel).\n\n## Performance and GPU fit\n\nElapsed **5.81 min**, 7 calls; 2364 generated tokens at **8.82 tok/s**, prefill 424.35 tok/s. Peak VRAM **10628/12227 MiB**; process-tree RSS **11863 MiB**; whole-system RAM **28577/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications. Shared GPU memory and paging were not measured.\n\n1 native CUDA launch, 65,536-token allocation, one slot, f16 KV, reasoning 4096. Dense FFNs for 39/64 layers on GPU and 25 on CPU: hybrid residency, not full GPU loading. No explicit endpoint failure, CUDA OOM or context overflow found in the final log. Maximum captured session fill 10.8%; maximum observed native released-slot size 8566 tokens (13.1%). These are sampled observations, not continuous KV high-water marks. No compaction events observed. Performance is separate from both fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: in packages/service/src/chat/manager.ts buildInstructions coding guidance, test the reusable prompt: “When aggregating arbitrary string keys, verify inherited-property names and preserve every original payload.” Prefer exercising the API contract over increasing token budgets; no truncation or budget pressure is observed. Promote a model-specific behavior only if diverse coding probes justify it.\n\n## Strategic experiment — not applied\n\nProposed experiment, not applied: add metamorphic arbitrary-kind probes to evals/src/scenarios/interface-contract.ts, including inherited names and repeated events, while retaining the existing source-independent integration boundary. In the product, expose bounded isolated execution checks so the model can discover the same class of issue before completion.\n\n## Generalization and next experiment\n\nR1 and R2 both score fixed 10 and both fail inherited-key counts. Compare any generic key-handling improvement against add-tests/state-transition and data-normalization; measure accepted behavior rather than matching a particular implementation or selector.\n\nRun any proposed experiment separately after the frozen campaign. No source, gate, model tuning, model file or runner state was changed. No capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/facts.json): pass, fixed 10, 3.35 min, 9.41 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/facts.json): pass, fixed 9.5, 0.88 min, 126.37 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-55-13-144Z-27lw/facts.json): pass, fixed 10, 0.83 min, 115.11 tok/s.\n\nOther-model history is contextual and not a controlled native-version comparison.\n\n## Strategic note\n\nRepeated fixed 10 scores conceal the same valid-input aggregation failure. The compiler and existing sample integration are necessary evidence, not exhaustive correctness guarantees.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json, historical-comparison.json and final captured files. Additional independent audits: advisory-endpoint-probes.json for Squisq; advisory-key-probes.json for interface-contract; advisory-docx-inspection.json and the preserved provisional snapshot for the Word report.\n" }, { "cellId": "r2-saluki-cuda-developer-craftbook-code-review", "baselineCellId": "r2-saluki-cuda-developer-craftbook-code-review", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T22:45:20.932Z", "finishedAt": "2026-11-05T23:02:01.620Z", "durationMs": 1000688, "success": true, "reason": "craftbook-code-review passed 4 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 803159 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-code-review", "bytes": 4257, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2529516860783146 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 803159 }, { "source": "facts", "path": "outcome.durationMs", "value": 1000688 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 168, "peakRssMb": 12884, "peakCpuPercent": 1204, "firstAtMs": 0, "lastAtMs": 986739 }, "gpu": { "available": true, "sampleCount": 168, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 168, "peakUsedMb": 29379, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 44713, "totalOutputTokens": 6973, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 30872, "todayTokensCached": 19712, "todayTokensOut": 5404, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 30872, "totalTokensCached": 19712, "totalTokensOut": 5404, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11205, "cachedInputTokens": 10785, "outputTokens": 88, "cost": 0, "durationMs": 131677, "outputTokensPerSec": 8.450063186075926, "at": "2026-11-05T23:01:02.264Z" }, "lastUpdated": "2026-11-05T23:01:02.264Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T23:01:55.741Z" } }, "source": "engine-log" }, "derived": { "durationMs": 988264, "meanTokensPerSec": 8.39, "genTokensPerSec": 8.39, "genTokens": 6973, "promptTokensPerSec": 416.66, "peakRssMb": 12884, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 30872, "output": 5404 } } } }, "craftbook": { "craftbookId": "code-review", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 2, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "code-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T22:45:28.850Z", "completedAt": "2026-11-05T22:47:42.358Z", "ms": 133508, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "f6b6e8a7-3c50-4806-a434-7c7114fa0a5a" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-05T22:47:42.359Z", "completedAt": "2026-11-05T22:58:44.605Z", "ms": 662246, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "f6b6e8a7-3c50-4806-a434-7c7114fa0a5a" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "done", "activatedAt": "2026-11-05T22:58:44.606Z", "completedAt": "2026-11-05T23:01:02.222Z", "ms": 137616, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "f6b6e8a7-3c50-4806-a434-7c7114fa0a5a" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T22:45:27.011Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T22:45:26.907Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 137616 }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "malgorzata": 2 }, "perTask": { "code-review-workflow-eval/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 2, "continuityBreaks": 0, "sessionsPerStep": 0.2, "maxMessages": 6, "maxContextFill": 0.1779632568359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1779632568359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-code-review passed 4 deterministic craftbook checks", "durationMs": 1000688, "timeoutMs": 3956044, "budgetUsedFraction": 0.2529516860783146 }, "timing": { "startedAt": "2026-11-05T22:45:20.932Z", "finishedAt": "2026-11-05T23:02:01.620Z", "timeToFirstArtifactMs": 803159, "timeToLastArtifactWriteMs": 803159, "timeToFirstTokenMs": 32600, "firstTurnTtftMs": 18824, "timeToFirstToolCallMs": 41447 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "read_artifact": 3, "write_task_note": 2, "advance_task_step": 2, "write_artifact": 1, "read_files": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 15 }, "sniff": { "progression": [ { "atMs": 8177, "filePath": "craftbook-code-review", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, src/payment.js, docs/review-conventions.md" }, { "atMs": 805148, "filePath": "craftbook-code-review", "bytes": 4257, "score": 2, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, src/payment.js, docs/review-conventions.md" }, { "atMs": 946502, "filePath": "craftbook-code-review", "bytes": 4257, "score": 3, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, src/payment.js, docs/review-conventions.md" }, { "atMs": 994309, "filePath": "craftbook-code-review", "bytes": 4257, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-code-review", "bytes": 4257, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T22:45:20.936Z] [trial] id=craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff scenario=craftbook-code-review model=underdog-saluki-27b-iq2-mix", "[2026-11-05T22:45:20.936Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-code-review-AHDKUK", "[2026-11-05T22:45:20.936Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "[2026-11-05T22:45:20.936Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T22:45:20.936Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T22:45:20.939Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T22:45:20.944Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T22:45:20.949Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-code-review-AHDKUK", "[2026-11-05T22:45:20.949Z] [trial] throughput-scaled ceiling: 30m → 66m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T22:45:26.996Z] [trial] daemon spawned pid=10268 port=https://127.0.0.1:63087", "[2026-11-05T22:45:28.347Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T22:45:28.363Z] [trial] meester=vanya", "[2026-11-05T22:45:28.363Z] [trial] running scenario setup hook", "[2026-11-05T22:45:29.021Z] [trial] scenario setup complete", "[2026-11-05T22:45:29.022Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T22:45:29.074Z] [poll] initial digests hard=503781f89f81bed9 soft=33c6ba986c1381e3", "[2026-11-05T22:45:29.154Z] [poll] hard-progress: 503781f89f81bed9 → 3f35654a2489073c after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T22:45:34.264Z] [poll] hard-progress: 3f35654a2489073c → a35f3d2cb6723006 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T22:46:03.648Z] [poll] hard-progress: a35f3d2cb6723006 → da664ea51d840c1e after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=30,stream=67) sniff=1/0B)", "[2026-11-05T22:46:15.454Z] [poll] hard-progress: da664ea51d840c1e → 53f9f4eb59540235 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=47,stream=124) sniff=1/0B)", "[2026-11-05T22:47:31.895Z] [poll] hard-progress: 53f9f4eb59540235 → 2b3583582b5578f9 after 76s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=242,stream=1203) sniff=1/0B)", "[2026-11-05T22:47:43.464Z] [poll] hard-progress: 2b3583582b5578f9 → 4924385b84f6b585 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=261,stream=1263) sniff=1/0B)", "[2026-11-05T22:58:46.139Z] [poll] hard-progress: 4924385b84f6b585 → 4fe77da6e8469072 after 663s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=5,slot=1993,stream=7353) sniff=2/4257B)", "[2026-11-05T22:59:33.466Z] [poll] hard-progress: 4fe77da6e8469072 → f52d5e54dc03f2db after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=6,slot=2034,stream=7447) sniff=2/4257B)", "[2026-11-05T23:00:49.949Z] [poll] hard-progress: f52d5e54dc03f2db → 48aee116797cfda7 after 76s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=7,slot=2237,stream=8337) sniff=2/4257B)" ] }, "metrics": { "process": { "sampleCount": 168, "peakRssMb": 12884, "peakCpuPercent": 1204, "firstAtMs": 0, "lastAtMs": 986739 }, "gpu": { "available": true, "sampleCount": 168, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 168, "peakUsedMb": 29379, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 44713, "totalOutputTokens": 6973, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 30872, "todayTokensCached": 19712, "todayTokensOut": 5404, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 30872, "totalTokensCached": 19712, "totalTokensOut": 5404, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11205, "cachedInputTokens": 10785, "outputTokens": 88, "cost": 0, "durationMs": 131677, "outputTokensPerSec": 8.450063186075926, "at": "2026-11-05T23:01:02.264Z" }, "lastUpdated": "2026-11-05T23:01:02.264Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-05T23:01:55.741Z" } }, "source": "engine-log" }, "derived": { "durationMs": 988264, "meanTokensPerSec": 8.39, "genTokensPerSec": 8.39, "genTokens": 6973, "promptTokensPerSec": 416.66, "peakRssMb": 12884, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 30872, "output": 5404 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T22:45:20.932Z", "finishedAt": "2026-11-05T23:02:01.620Z", "timeToFirstArtifactMs": 803159, "timeToLastArtifactWriteMs": 803159, "timeToFirstTokenMs": 32600, "firstTurnTtftMs": 18824, "timeToFirstToolCallMs": 41447 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 2, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "code-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T22:45:28.850Z", "completedAt": "2026-11-05T22:47:42.358Z", "ms": 133508, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "f6b6e8a7-3c50-4806-a434-7c7114fa0a5a" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-05T22:47:42.359Z", "completedAt": "2026-11-05T22:58:44.605Z", "ms": 662246, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "f6b6e8a7-3c50-4806-a434-7c7114fa0a5a" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "done", "activatedAt": "2026-11-05T22:58:44.606Z", "completedAt": "2026-11-05T23:01:02.222Z", "ms": 137616, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "f6b6e8a7-3c50-4806-a434-7c7114fa0a5a" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T22:45:27.011Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T22:45:26.907Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 137616 }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "malgorzata": 2 }, "perTask": { "code-review-workflow-eval/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 2, "continuityBreaks": 0, "sessionsPerStep": 0.2, "maxMessages": 6, "maxContextFill": 0.1779632568359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1779632568359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-craftbook-code-review", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "scenarioId": "craftbook-code-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-05T23:08:01.639Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "artifactBytes": 4269, "artifactSha256": "d1bc463d08f5b54d8214c4af82ca95e5f2e2aef9e10aa4cb0659f040c8f52925", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/co/code-review/versions/1.0.4/test.json", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": { "workflow fidelity": 7, "usefulness": 6 }, "meanScore": 6.5, "justification": "The report finds all four core defects and gives concrete fixes with a consistent request-changes verdict. It also lists two explicitly acceptable house patterns as findings, contrary to the conventions. Required source reads happen only after terminal completion and do not revise the report. Input validation is partly redundant with the injection finding.", "reviewBasis": "Read the full 4,269-byte report, all seven-file staged diff hunks, manifest, review conventions, payment source and nine tool receipts. Verified the original four defects are covered and the two house-style nits remain in the final Findings table. Required input reads were triggered by the final sniff only after the workflow completed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "bytes": 4269, "sha256": "d1bc463d08f5b54d8214c4af82ca95e5f2e2aef9e10aa4cb0659f040c8f52925", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff", "bytes": 4396, "sha256": "a1744bbef7b2c7cbd7ab3feba3df59a32a472f32781773715a67b8525758b728", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "bytes": 1113, "sha256": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/docs/review-conventions.md", "bytes": 1389, "sha256": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/src/payment.js", "bytes": 596, "sha256": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 1372, "sha256": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:36.626Z", "reviewId": "071", "cellId": "r2-saluki-cuda-developer-craftbook-code-review", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "scenarioId": "craftbook-code-review", "status": "scored", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "baselineCellId": "r2-saluki-cuda-developer-craftbook-code-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b4aa71422a4c8b21d7cfe552565c7e2160bf094dfb5206deb49b01f7698fec66", "evidenceSha256": "12a445cb84c8f5c4540ada29d128ea035f4e9759936a2f47f057411b79e2274e", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "bytes": 4269, "sha256": "d1bc463d08f5b54d8214c4af82ca95e5f2e2aef9e10aa4cb0659f040c8f52925", "shownBytes": 4269, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "bytes": 1113, "sha256": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "shownBytes": 1113, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/docs/review-conventions.md", "bytes": 1389, "sha256": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b", "shownBytes": 1389, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/src/payment.js", "bytes": 596, "sha256": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee", "shownBytes": 596, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 1372, "sha256": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "shownBytes": 1372, "truncated": false } ], "contextFiles": [], "scoreAxes": { "workflow fidelity": 6, "usefulness": 7 }, "justification": "The report has Summary, a severity-ranked Findings table and a Verdict, with accurate payment.js line cites (6, 7, 9). It catches the SQL injection, pagination, leak and catch-success bugs. But it lists two deliberate house-style patterns (audit, `== null`) as nit findings, which the conventions forbid raising.", "limitations": "The scope note, task-note stamp and step advancement were not shown, so the scope and done steps can't be checked. changes.diff wasn't supplied, so line cites outside payment.js are unverified. The errors.js arity pattern was handled by omission, which I couldn't verify.", "meanScore": 6.5, "durationMs": 16445, "costUsd": 0.030871799999999998, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6180, "cache_read_input_tokens": 778, "output_tokens": 607, "output_tokens_details": { "thinking_tokens": 265 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6180, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 607, "cache_read_input_tokens": 778, "cache_creation_input_tokens": 6180, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6180 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 607, "cacheReadInputTokens": 778, "cacheCreationInputTokens": 6180, "webSearchRequests": 0, "costUSD": 0.030871799999999998, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 265, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6180, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-code-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 16m 41s; budget used: 25.3%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 803159`; `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":4257,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.2529516860783146`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 803159`; `facts.outcome.durationMs: 1000688` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12884 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29379 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.39 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 44713 + 6973 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 138s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.20 | `continuity.sessions` |\n| Sessions reused across steps | 1 (2 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 18% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 803159`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":4257,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.2529516860783146`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 803159`\n- `facts.outcome.durationMs: 1000688`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 803159`; `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":4257,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.2529516860783146`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 803159`; `facts.outcome.durationMs: 1000688`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory review\n\nFixed result **pass 10/10** is unchanged. Declared advisory axes: workflow fidelity 7/10; usefulness 6/10. Mean **6.50/10**, codex-inline/current-session.\n\nThe report finds all four core defects and gives concrete fixes with a consistent request-changes verdict. It also lists two explicitly acceptable house patterns as findings, contrary to the conventions. Required source reads happen only after terminal completion and do not revise the report. Input validation is partly redundant with the injection finding.\n\nRead the full 4,269-byte report, all seven-file staged diff hunks, manifest, review conventions, payment source and nine tool receipts. Verified the original four defects are covered and the two house-style nits remain in the final Findings table. Required input reads were triggered by the final sniff only after the workflow completed.\n\n## Positives\n\n- The report covers SQL injection, the pagination off-by-one, export handle leakage on errors, and settings error masking. Its request-changes verdict and concrete remedies are consistent with those defects (outcome.success; sniff.latest full pass; artifacts.otherFileCount=15; final report versus changes.diff).\n- The actual code-review workflow completes scope, report and done with two gate approvals and no rejection, modifies no project source, and uses only nine recorded tools (continuity.steps.completed=3; continuity.steps.gateApprovals=2; toolUse.byTool; final workspace versus seeded fixtures).\n\n## Negatives\n\n- Findings 7 and 8 explicitly describe best-effort audit and == null as acceptable, yet still classify them as nits. The conventions explicitly forbid raising these patterns as findings. All four fixed checks still pass; neither a separate findings.json nor an oracle execution receipt is present (sniff.latest.score=4/4; artifacts.otherFileCount=15; report, conventions and tool trace).\n- The reviewer writes and stamps the report before reading the required brief, payment source and conventions. At 23:01:55Z, after workflow completion at 23:01:02Z, a single read_files call satisfies the last input-read sniff. There is no later report revision, so the final read requirement proves access but not evidence-informed review (toolUse.byTool.read_files=1, write_artifact=1; outcome.reason; log.txt and advisory-tool-evidence.json).\n- The amount-validation issue is legitimate context, but calling it the root enabler of injection conflates validation with safe SQL binding and partly duplicates the critical finding. The no-validation behavior was already present in the removed code; the report should distinguish pre-existing risk from introduced regression (artifacts.otherFileCount; payment diff; final report).\n\n## Performance and GPU fit\n\nElapsed **16.68 min**, 9 calls; 6973 generated tokens at **8.39 tok/s**, prefill 416.66 tok/s. Peak VRAM **10630/12227 MiB**; process-tree RSS **12884 MiB**; whole-system RAM **29379/31865 MiB** (perf.derived, perf.gpu, perf.process, perf.systemMemory). Whole-system memory includes other applications. Shared GPU memory and paging were not measured.\n\n1 native CUDA launch, 65,536-token allocation, one slot, f16 KV, reasoning 4096. Dense FFNs for 39/64 layers on GPU and 25 on CPU: hybrid residency, not full GPU loading. No explicit endpoint failure, CUDA OOM or context overflow found in the final log. Maximum captured session fill 17.8%; maximum observed native released-slot size 15323 tokens (23.4%). These are sampled observations, not continuous KV high-water marks. No compaction events observed. Performance is separate from both fixed and advisory quality.\n\n## Tactical experiment — not applied\n\nProposed experiment, not applied: in Gilde code-review craftbook.json based on 1.0.4, scope/report prompts should require a “cleared observations” section outside Findings and separate introduced defects from pre-existing concerns. Read declared review conventions before classifying any issue. Do not encode the fixture-specific four defects or bait names in model tuning.\n\n## Strategic experiment — not applied\n\nProposed experiment, not applied: in the reusable craftbook input-read check, require relevant read evidence to precede the first substantive report write; after a late read, require review or a content-version acknowledgment before accepting evidence-informed completion. Separately validate structured findings against explicitly cleared observations. An oracle fixture in setup is not evidence of execution; any new predicate needs a separate eval revision after this campaign.\n\n## Generalization and next experiment\n\nR1 fixed 9.5/advisory 7.5 avoided house-style bait but had line-citation errors; R2 fixed 10/advisory 6.5 improves citations while adding false-positive nits and late reads. Compare source-before-write and cleared-observation experiments with the unrelated bibliography and API review tasks; measure precision as well as recall.\n\nRun any proposed experiment separately after the frozen campaign. No source, gate, model tuning, model file or runner state was changed. No capability failure was retried.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/facts.json): pass, fixed 9.5, 26.56 min, 8.18 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/facts.json): pass, fixed 9.5, 2.26 min, 120.75 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T11-58-56-299Z-1y5b/facts.json): pass, fixed 9.5, 1.74 min, 124.16 tok/s.\n\nOther-model history is contextual and not a controlled native-version comparison.\n\n## Strategic note\n\nA higher fixed score here reflects efficient checked workflow completion, not uniformly better review usefulness. The report detects real defects but mixes cleared patterns into the finding count, which can burden a human reviewer.\n\n## Raw evidence\n\nSee facts.json, score.json, advisory-artifact-review.json, advisory-tool-evidence.json, historical-comparison.json and final captured files. Additional independent audits: advisory-endpoint-probes.json for Squisq; advisory-key-probes.json for interface-contract; advisory-docx-inspection.json and the preserved provisional snapshot for the Word report.\n" }, { "cellId": "r2-saluki-cuda-developer-craftbook-api-contract-review", "baselineCellId": "r2-saluki-cuda-developer-craftbook-api-contract-review", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-05T23:05:09.446Z", "finishedAt": "2026-11-06T00:07:49.151Z", "durationMs": 3759705, "success": true, "reason": "craftbook-api-contract-review passed 6 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1009639 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-api-contract-review", "bytes": 11648, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.9503698644403349 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 24 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1009639 }, { "source": "facts", "path": "outcome.durationMs", "value": 3759705 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" }, "facts": { "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 630, "peakRssMb": 14525, "peakCpuPercent": 1217.5, "firstAtMs": 1, "lastAtMs": 3747687 }, "gpu": { "available": true, "sampleCount": 630, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 630, "peakUsedMb": 31380, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 83024, "totalOutputTokens": 30619, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 27851, "todayTokensCached": 22723, "todayTokensOut": 1646, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 82976, "totalTokensCached": 66897, "totalTokensOut": 17778, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 6961, "cachedInputTokens": 6670, "outputTokens": 745, "cost": 0, "durationMs": 135843, "outputTokensPerSec": 9.028703267055693, "at": "2026-11-06T00:07:42.495Z" }, "lastUpdated": "2026-11-06T00:07:42.495Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T00:07:44.773Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3749024, "meanTokensPerSec": 8.82, "genTokensPerSec": 8.82, "genTokens": 30619, "promptTokensPerSec": 400.83, "peakRssMb": 14525, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 82976, "output": 17778 } } } }, "craftbook": { "craftbookId": "api-contract-review", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "api-contract-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T23:05:17.075Z", "completedAt": "2026-11-05T23:21:59.262Z", "ms": 1002187, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "56564c99-3370-48d0-8f68-b458a31b79a2" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "review", "activatedAt": "2026-11-05T23:21:59.264Z", "completedAt": "2026-11-05T23:48:41.085Z", "ms": 1601821, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "9f059d95-f1a3-4fa8-ba71-8fb01e441fe7" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-05T23:48:41.087Z", "completedAt": "2026-11-05T23:59:12.895Z", "ms": 631808, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "bc49eaf7-f83e-4a3d-8d89-3693776d4b6a" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T23:59:12.896Z", "completedAt": "2026-11-06T00:05:20.011Z", "ms": 367115, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "bc49eaf7-f83e-4a3d-8d89-3693776d4b6a" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "finish", "activatedAt": "2026-11-06T00:05:20.013Z", "completedAt": "2026-11-06T00:07:42.441Z", "ms": 142428, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "c28b333b-671d-4beb-8afe-9896b1db5260" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T23:05:15.289Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T23:05:15.192Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 631808 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "eve": 2, "santiago": 2 }, "perTask": { "api-contract-review-workflow-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.22808837890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.22808837890625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-api-contract-review passed 6 deterministic craftbook checks", "durationMs": 3759705, "timeoutMs": 3956044, "budgetUsedFraction": 0.9503698644403349 }, "timing": { "startedAt": "2026-11-05T23:05:09.446Z", "finishedAt": "2026-11-06T00:07:49.151Z", "timeToFirstArtifactMs": 1009639, "timeToLastArtifactWriteMs": 3610484, "timeToFirstTokenMs": 33892, "firstTurnTtftMs": 20252, "timeToFirstToolCallMs": 59198 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 24, "byTool": { "read_files": 4, "write_artifact": 5, "read_file": 7, "read_artifact": 3, "read_task_notes": 1, "list_dir": 1, "read_artifacts": 1, "write_task_note": 1, "advance_task_step": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 11 }, "sniff": { "progression": [ { "atMs": 7879, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, openapi.yaml, src/routes.ts, src/session-routes.ts, docs/api-contract.md" }, { "atMs": 60155, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: src/routes.ts, src/session-routes.ts" }, { "atMs": 71948, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 2, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 1013457, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 2611928, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 4, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 3248778, "filePath": "craftbook-api-contract-review", "bytes": 11648, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 3754837, "filePath": "craftbook-api-contract-review", "bytes": 11648, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-api-contract-review", "bytes": 11648, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-05T23:05:09.450Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-api-contract-review-W9hYbn", "[2026-11-05T23:05:09.449Z] [trial] id=craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b scenario=craftbook-api-contract-review model=underdog-saluki-27b-iq2-mix", "[2026-11-05T23:05:09.450Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-05T23:05:09.450Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "[2026-11-05T23:05:09.450Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-05T23:05:09.452Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-05T23:05:09.456Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-05T23:05:09.462Z] [trial] throughput-scaled ceiling: 30m → 66m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-05T23:05:09.462Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-api-contract-review-W9hYbn", "[2026-11-05T23:05:15.249Z] [trial] daemon spawned pid=2332 port=https://127.0.0.1:53427", "[2026-11-05T23:05:16.651Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-05T23:05:16.669Z] [trial] running scenario setup hook", "[2026-11-05T23:05:16.669Z] [trial] meester=sulien", "[2026-11-05T23:05:17.234Z] [trial] scenario setup complete", "[2026-11-05T23:05:17.235Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-05T23:05:17.284Z] [poll] initial digests hard=8c0c3d4c6571dd95 soft=8ae59ff902a65afe", "[2026-11-05T23:05:17.370Z] [poll] hard-progress: 8c0c3d4c6571dd95 → 75e0e6baf746f762 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T23:05:22.501Z] [poll] hard-progress: 75e0e6baf746f762 → a3434ea5cf901f20 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-05T23:06:09.657Z] [poll] hard-progress: a3434ea5cf901f20 → 2db114b86bceaeca after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=77,stream=229) sniff=1/0B)", "[2026-11-05T23:06:21.451Z] [poll] hard-progress: 2db114b86bceaeca → b26c48b0350c17a8 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=94,stream=290) sniff=2/0B)", "[2026-11-05T23:06:33.158Z] [poll] hard-progress: b26c48b0350c17a8 → 6f86f55cd4b49306 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=114,stream=361) sniff=2/0B)", "[2026-11-05T23:16:16.772Z] [poll] hard-progress: 6f86f55cd4b49306 → 8d9077feaa277385 after 584s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1789,stream=6822) sniff=2/0B)", "[2026-11-05T23:17:46.302Z] [poll] hard-progress: 8d9077feaa277385 → a989f8f5408a8f93 after 90s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=2025,stream=7527) sniff=2/0B)", "[2026-11-05T23:22:02.966Z] [poll] hard-progress: a989f8f5408a8f93 → c0af37b1b2128b00 after 257s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=6,slot=2753,stream=11246) sniff=3/0B)", "[2026-11-05T23:22:50.474Z] [poll] hard-progress: c0af37b1b2128b00 → 8f0c7be65f862187 after 48s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=10,slot=2793,stream=11340) sniff=3/0B)" ] }, "metrics": { "process": { "sampleCount": 630, "peakRssMb": 14525, "peakCpuPercent": 1217.5, "firstAtMs": 1, "lastAtMs": 3747687 }, "gpu": { "available": true, "sampleCount": 630, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 630, "peakUsedMb": 31380, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 83024, "totalOutputTokens": 30619, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 27851, "todayTokensCached": 22723, "todayTokensOut": 1646, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 82976, "totalTokensCached": 66897, "totalTokensOut": 17778, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 6961, "cachedInputTokens": 6670, "outputTokens": 745, "cost": 0, "durationMs": 135843, "outputTokensPerSec": 9.028703267055693, "at": "2026-11-06T00:07:42.495Z" }, "lastUpdated": "2026-11-06T00:07:42.495Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T00:07:44.773Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3749024, "meanTokensPerSec": 8.82, "genTokensPerSec": 8.82, "genTokens": 30619, "promptTokensPerSec": 400.83, "peakRssMb": 14525, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 82976, "output": 17778 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-05T23:05:09.446Z", "finishedAt": "2026-11-06T00:07:49.151Z", "timeToFirstArtifactMs": 1009639, "timeToLastArtifactWriteMs": 3610484, "timeToFirstTokenMs": 33892, "firstTurnTtftMs": 20252, "timeToFirstToolCallMs": 59198 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "api-contract-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-05T23:05:17.075Z", "completedAt": "2026-11-05T23:21:59.262Z", "ms": 1002187, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "56564c99-3370-48d0-8f68-b458a31b79a2" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "review", "activatedAt": "2026-11-05T23:21:59.264Z", "completedAt": "2026-11-05T23:48:41.085Z", "ms": 1601821, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "9f059d95-f1a3-4fa8-ba71-8fb01e441fe7" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-05T23:48:41.087Z", "completedAt": "2026-11-05T23:59:12.895Z", "ms": 631808, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "bc49eaf7-f83e-4a3d-8d89-3693776d4b6a" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-05T23:59:12.896Z", "completedAt": "2026-11-06T00:05:20.011Z", "ms": 367115, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "bc49eaf7-f83e-4a3d-8d89-3693776d4b6a" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "finish", "activatedAt": "2026-11-06T00:05:20.013Z", "completedAt": "2026-11-06T00:07:42.441Z", "ms": 142428, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "c28b333b-671d-4beb-8afe-9896b1db5260" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-05T23:05:15.289Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-05T23:05:15.192Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 631808 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "eve": 2, "santiago": 2 }, "perTask": { "api-contract-review-workflow-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.22808837890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.22808837890625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "scenarioId": "craftbook-api-contract-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-06T00:09:51.409Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "artifactBytes": 11740, "artifactSha256": "e8713bf3d370d92787bbd1a4b5969e00312765d8cdcb443a49fcfce18bcbc746", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/ap/api-contract-review/versions/1.1.5/test.json", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": { "workflow fidelity": 7, "usefulness": 5 }, "meanScore": 6, "justification": "All three real drifts are covered and the safe lookalikes are cleared. The history fix still promises an unimplemented200 route, the API-key security snippet is invalid, and compatibility/auth conclusions exceed the supplied evidence. Staged artifacts and concrete snippets help, but the reviewer signs off without correcting these issues.", "reviewBasis": "Read the complete11,740-byte report,18,237-byte endpoint review,6,435-byte scope, reviewer verdict and five supplied input files. Final captured files match the inspected snapshot hashes. All11 YAML snippets parse; a targeted official-OAS3.0.3 check finds the invalid API-key scope list. The model read the harness oracle. Five stages completed and the standard trial passed6/6; no model rerun or score mutation.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "bytes": 11740, "sha256": "e8713bf3d370d92787bbd1a4b5969e00312765d8cdcb443a49fcfce18bcbc746", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "bytes": 6435, "sha256": "f81e5302bf5771d904723d1a04f4baf35e07958db1627b9e68287aa7eafd9498", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "bytes": 18237, "sha256": "56b3778006a3a9e5f1446e86fe709aef695c5068248351e2d4d7f17139a33b61", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/verdict.md", "bytes": 1603, "sha256": "89398172ee9176bbdbfc473ac534937db93c6a699a12b57365b96b24a0ab3982", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/openapi.yaml", "bytes": 3167, "sha256": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/src/routes.ts", "bytes": 1146, "sha256": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/src/session-routes.ts", "bytes": 176, "sha256": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/docs/api-contract.md", "bytes": 866, "sha256": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 2833, "sha256": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "status": "reviewedSupportingArtifact" } ], "observations": [ "F3 correctly detects the missing history handler, but adding a 404 response while retaining the promised 200 does not implement the operation. The prose presents adding a handler as an alternative to the insufficient snippet.", "The apiKey snippet uses [widget_api_key] as a scope list for an apiKey scheme; the OAS3.0.3 security requirement must use an empty array for this scheme type.", "Supporting review.md incorrectly infers all Widget fields are required because requestBody.required=true. Its F6 snippet requires display_name but declares displayName; the final primary fixes that naming inconsistency but still assumes required fields without implementation evidence.", "Final report calls F2 a contract/implementation disagreement although both currently use200. A better create response and response schema are design/incompleteness findings, distinct from the three actual reconciliation mismatches.", "No auth in the supplied handlers does not establish that deployment/global middleware makes the API public. Stored user/session rows are not shown, so claims of exact response-schema matches are stronger than evidence.", "All proposed changes are grouped under a mandatory2.0.0 bump. No prior contract, deployment policy or consumer evidence is supplied; a metadata bump alone does not implement concurrent versions or migration.", "At23:23:14Z the model read the harness precision/recall oracle, including text describing the three real drifts and three safe lookalikes. Recall cannot be treated as fully independent of grader guidance." ], "health": { "taskStatus": "complete", "activeStepId": null, "maxObservedReleasedSlotTokens": 26835, "allocatedTokens": 65536, "nativeLaunches": 1, "nativeErrorLines": [], "gpuPointMiB": 10630, "gpuTotalMiB": 12227, "systemUsedPointMiB": 29280, "latestObservedDecodeTokensPerSec": 8.92, "note": "Point/sampled observations, not continuous peaks. Dense FFNs39/64GPU,25CPU; f16KV. Campaign left unchanged.", "elapsedMs": 3759705, "generatedTokens": 30619, "meanTokensPerSec": 8.82, "promptTokensPerSec": 400.83, "peakVramMiB": 10642, "peakProcessRssMiB": 14525, "peakSystemRamMiB": 31380, "capturedSessionMaxContextFill": 0.22808837890625 } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:39.652Z", "reviewId": "072", "cellId": "r2-saluki-cuda-developer-craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "scenarioId": "craftbook-api-contract-review", "status": "scored", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "baselineCellId": "r2-saluki-cuda-developer-craftbook-api-contract-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "7b82466469420715d4581609377eeda15172b59c669f4f99c7750de33add749d", "evidenceSha256": "46b85a5a2fe754b87628574ad03add27a4a28770d52e4f3bdea5853c24ba6071", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "bytes": 11740, "sha256": "e8713bf3d370d92787bbd1a4b5969e00312765d8cdcb443a49fcfce18bcbc746", "shownBytes": 11740, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "bytes": 6435, "sha256": "f81e5302bf5771d904723d1a04f4baf35e07958db1627b9e68287aa7eafd9498", "shownBytes": 6435, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "bytes": 18237, "sha256": "56b3778006a3a9e5f1446e86fe709aef695c5068248351e2d4d7f17139a33b61", "shownBytes": 18237, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/verdict.md", "bytes": 1603, "sha256": "89398172ee9176bbdbfc473ac534937db93c6a699a12b57365b96b24a0ab3982", "shownBytes": 1603, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/openapi.yaml", "bytes": 3167, "sha256": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a", "shownBytes": 3167, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/src/routes.ts", "bytes": 1146, "sha256": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453", "shownBytes": 1146, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/src/session-routes.ts", "bytes": 176, "sha256": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8", "shownBytes": 176, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/docs/api-contract.md", "bytes": 866, "sha256": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1", "shownBytes": 866, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md", "bytes": 2833, "sha256": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "shownBytes": 2833, "truncated": false } ], "contextFiles": [], "scoreAxes": { "workflow fidelity": 8, "usefulness": 8 }, "justification": "Scope, review and report artifacts exist with a locked C1–C8 checklist, per-operation review, and verdict plus corrected snippets. It handles the traps: the owner nullable email, session-routes.ts, and a real drift on /widgets/:id. Flaws: \"— impact\" heading artifacts, no per-finding severity in the final report, and required [id,…] on a create body.", "limitations": "Only the artifacts shown were reviewed. Gate execution, advance_task_step calls and the evaluate/finish steps were not observed. The verdict.md self-grades its own work. The snippets were not validated against an OpenAPI linter.", "meanScore": 8, "durationMs": 18626, "costUsd": 0.0938418, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 21360, "cache_read_input_tokens": 778, "output_tokens": 832, "output_tokens_details": { "thinking_tokens": 487 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 21360, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 832, "cache_read_input_tokens": 778, "cache_creation_input_tokens": 21360, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 21360 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 832, "cacheReadInputTokens": 778, "cacheCreationInputTokens": 21360, "webSearchRequests": 0, "costUSD": 0.0938418, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 487, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 21360, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-api-contract-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 62m 40s; budget used: 95.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1009639`; `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":11648,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.9503698644403349`; `facts.toolUse.totalToolCalls: 24`; `facts.timing.timeToFirstArtifactMs: 1009639`; `facts.outcome.durationMs: 3759705` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 5 + 0.15 * 10 = **9.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14525 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31380 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.82 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 83024 + 30619 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 632s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 23% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1009639`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":11648,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.9503698644403349`\n- `facts.toolUse.totalToolCalls: 24`\n- `facts.timing.timeToFirstArtifactMs: 1009639`\n- `facts.outcome.durationMs: 3759705`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1009639`; `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":11648,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.9503698644403349`; `facts.toolUse.totalToolCalls: 24`; `facts.timing.timeToFirstArtifactMs: 1009639`; `facts.outcome.durationMs: 3759705`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 9/10** remains unchanged. Advisory **6.0/10**: workflow fidelity7/10, usefulness5/10, codex-inline/current-session.\n\nAll three real drifts are covered and the safe lookalikes are cleared. The history fix still promises an unimplemented200 route, the API-key security snippet is invalid, and compatibility/auth conclusions exceed the supplied evidence. Staged artifacts and concrete snippets help, but the reviewer signs off without correcting these issues.\n\nRead the complete11,740-byte report,18,237-byte endpoint review,6,435-byte scope, reviewer verdict and five supplied input files. Final captured files match the inspected snapshot hashes. All11 YAML snippets parse; a targeted official-OAS3.0.3 check finds the invalid API-key scope list. The model read the harness oracle. Five stages completed and the standard trial passed6/6; no model rerun or score mutation.\n\n## Positives\n\n- All6 deterministic checks pass, all5 stages complete, four automatic gate approvals are recorded, and all3 declared artifacts are produced. Supplied contract/convention/route inputs are read before the first saved scope (outcome.success; sniff.latest; continuity.steps.completed=5; continuity.steps.gateApprovals=4; toolUse.byTool; tool receipts).\n- The report finds the missing history implementation, undocumented single-widget route and retryCount string/number mismatch. It correctly clears optional owner-email omission, colon-versus-brace syntax and the session route registered through another module (artifacts.otherFileCount=11; final report against openapi.yaml and both route files).\n- The output has prioritized findings, concrete snippets and an explicit compatibility verdict. All11 snippets parse as YAML, although that does not establish their semantics (sniff.latest full pass; advisory-snippet-audit.json).\n\n## Negatives\n\n- The proposed history correction retains200 and adds404 while leaving the handler absent; the prose treats implementing the operation as an alternative to this insufficient snippet. This does not resolve the identified contract over-promise (artifacts.otherFileCount; final report F3 versus routes.ts/session-routes.ts).\n- The proposed apiKey requirement uses a nonempty scope list, which is invalid for that scheme type under OpenAPI3.0.3. Supporting review.md also confuses required request-body presence with required properties and contains a display_name/displayName mismatch in its draft F6 snippet; the final primary corrects the latter naming mismatch only (artifacts.otherFileCount; advisory-snippet-audit.json; final/supporting artifacts).\n- The report calls F2 a contract/implementation disagreement although both use200; documentation/design improvements are distinct from the three actual drifts. Opaque database-return types and absence of auth in the supplied handlers are overstated as exact response conformance and proven public deployment (artifacts.otherFileCount; final report and source files).\n- A blanket mandatory2.0.0 rule substitutes for a consumer/deployment migration analysis. No prior contract baseline or deployment policy is supplied; changing document metadata alone does not implement version routing. The final reviewer marks all7 criteria PASS without resolving these limitations (toolUse.byTool.write_artifact=5; verdict.md; scope.md; final report).\n- At23:23:14Z a read_file call exposes the harness precision/recall oracle, including its three-real-drift/three-safe-operation description. The workflow reads actual source too, but its recall is not independent of grader guidance. No claim of unseen or unaided defect discovery should be made (toolUse.byTool.read_file=7; advisory-tool-evidence.json).\n- The initial scope write is rejected because proposed201/204 checklist numbers are absent from source evidence; it succeeds after rewording. This write-guard rejection is separate from continuity.steps.gateRejections=0. The trial takes62m40s,24calls and95.0% of its budget without native failure (toolUse.byTool.write_artifact; outcome.durationMs; outcome.budgetUsedFraction; perf.derived; first rejected write receipt).\n\nThe security scope-array rule is defined in [OpenAPI3.0.3 Security Requirement Object](https://spec.openapis.org/oas/v3.0.3.html#security-requirement-object). Body-presence requiredness and document-version metadata are defined in its [Request Body Object](https://spec.openapis.org/oas/v3.0.3.html#request-body-object) and [Info Object](https://spec.openapis.org/oas/v3.0.3.html#info-object). These references were consulted by the independent monitor only, not injected into the model or used to change the frozen campaign.\n\n## Performance and GPU fit\n\nElapsed **62m40s**,24calls,30,619 engine-recorded generated tokens at **8.82tok/s**, prefill **400.83tok/s** (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10,642/12,227MiB**, process-tree RSS **14,525MiB**, whole-system RAM **31,380/31,865MiB** (perf.gpu; perf.process; perf.systemMemory). Whole-system RAM includes other apps; shared GPU memory and paging were not measured.\n\nOne native0.1.57 CUDA launch,65,536context,one slot,f16KV,reasoning4096; dense FFNs39/64GPU and25CPU. This is hybrid residency. Maximum captured session fill **22.8%**; maximum observed released native slot **26,835tokens (40.9%)**. Neither is a continuous KV peak. No compaction, endpoint failure, explicit CUDAOOM or context-overflow event. No new campaign hardware peak. Successful engine execution is distinct from review correctness.\n\n## Tactical experiment — not applied\n\nIn Gilde api-contract-review craftbook.json based on1.1.5, steps[id=review].prompt and steps[id=evaluate].prompt, test: “Label observed contract mismatch, unresolved evidence, and design proposal separately. A correction must remove the detected mismatch. Do not infer deployed auth or opaque returned-row types from an incomplete handler excerpt.” Add a requirement to check corrected snippets against the stated spec version. Do not encode fixture paths, expected drift counts or answers into model tuning.\n\n## Strategic experiments — not applied\n\nIn packages/core/src/checks/claims.ts distinguish proposed design rules from assertions about supplied source; measure false write rejections and unsupported assertions together. Add schema/semantic validation of suggested contract snippets as a reusable checker, beyond YAML syntax.\n\nThe harness-only oracle was readable through the ordinary model tool surface. Isolate evaluator-only files from model-readable workspace paths in evals/src while leaving genuine application tests available. Verify both the filesystem boundary and tool read denial with separate evaluations; never silently remove the observed exposure from this run.\n\n## Generalization and next experiment\n\nThe first repetition fixed9/advisory6.5 missed the history drift; this repetition fixed9/advisory6 improves that coverage while supplying inadequate fixes and unsupported conclusions. Oracle exposure limits independent-recall interpretation. No native, tuning or gate change occurred, so do not infer a causal build improvement.\n\nAfter this frozen campaign, test proposal/fact separation and corrected-snippet validation against API review, unrelated code review, and A/B statistics. Measure recall, false positives, validity of proposed fixes, false write rejections and added runtime. A report can detect a fault yet propose a correction that does not repair it.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/facts.json): pass, fixed9, 55.16min, 8.46tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/facts.json): pass, fixed8.5, 14.00min, 118.59tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-07-38-331Z-v8fm/facts.json): pass, fixed8.5, 5.04min, 121.8tok/s.\n\nOther-model history is contextual, not a controlled native-version comparison.\n\n## Strategic note\n\nAll five workflow phases and six fixed checks completed, but the reviewer approved invalid/inadequate corrections. Oracle access is an additional evaluation-isolation caveat. No model retry, source edit, tuning change or gate change was made.\n\n## Raw evidence\n\nfacts.json,score.json,advisory-artifact-review.json,advisory-snippet-audit.json,advisory-tool-evidence.json,historical-comparison.json and the preserved provisional snapshot. Final artifact hashes match the reviewed snapshot.\n" }, { "cellId": "r2-saluki-cuda-developer-craftbook-deep-security-review", "baselineCellId": "r2-saluki-cuda-developer-craftbook-deep-security-review", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T00:11:01.048Z", "finishedAt": "2026-11-06T00:44:26.601Z", "durationMs": 2005553, "success": true, "reason": "craftbook-deep-security-review passed 11 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 736330 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 8141, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.33797281661511686 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 28 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 736330 }, { "source": "facts", "path": "outcome.durationMs", "value": 2005553 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" }, "facts": { "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 335, "peakRssMb": 15044, "peakCpuPercent": 1232.3, "firstAtMs": 0, "lastAtMs": 1988658 }, "gpu": { "available": true, "sampleCount": 335, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 335, "peakUsedMb": 31275, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 56370, "totalOutputTokens": 16019, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 36168, "todayTokensCached": 22546, "todayTokensOut": 3523, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 36168, "totalTokensCached": 22546, "totalTokensOut": 3523, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11041, "cachedInputTokens": 9456, "outputTokens": 755, "cost": 0, "durationMs": 121430, "outputTokensPerSec": 9.056509206241891, "at": "2026-11-06T00:44:16.382Z" }, "lastUpdated": "2026-11-06T00:44:16.382Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T00:44:17.377Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1990189, "meanTokensPerSec": 8.96, "genTokensPerSec": 8.96, "genTokens": 16019, "promptTokensPerSec": 402.85, "peakRssMb": 15044, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 36168, "output": 3523 } } } }, "craftbook": { "craftbookId": "deep-security-review", "tasksObserved": 2, "craftbookTasks": 0, "reachedTerminal": false, "verdict": "artifact-only" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T00:11:06.739Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T00:11:06.632Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "lindiwe": 1, "rae": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 7, "maxContextFill": 0.1703338623046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1703338623046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-deep-security-review passed 11 deterministic craftbook checks", "durationMs": 2005553, "timeoutMs": 5934066, "budgetUsedFraction": 0.33797281661511686 }, "timing": { "startedAt": "2026-11-06T00:11:01.048Z", "finishedAt": "2026-11-06T00:44:26.601Z", "timeToFirstArtifactMs": 736330, "timeToLastArtifactWriteMs": 1995314, "timeToFirstTokenMs": 31488, "firstTurnTtftMs": 21173, "timeToFirstToolCallMs": 47708 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 28, "byTool": { "read_file": 14, "list_dir": 1, "make_dir": 1, "write_file": 7, "validate": 1, "run_installed_script": 1, "stat": 2, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 7758, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 0, "scoreMax": 11, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: src/admin.ts, src/profile.ts, src/reports.ts, docs/security.md" }, { "atMs": 48408, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 0, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 738100, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 2, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 917444, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 4, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 1405979, "filePath": "craftbook-deep-security-review", "bytes": 3704, "score": 9, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 3704 bytes matches. Add that content." }, { "atMs": 1590496, "filePath": "craftbook-deep-security-review", "bytes": 5062, "score": 10, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 5062 bytes matches. Add that content." }, { "atMs": 1775184, "filePath": "craftbook-deep-security-review", "bytes": 5081, "score": 10, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 5081 bytes matches. Add that content." }, { "atMs": 1995956, "filePath": "craftbook-deep-security-review", "bytes": 8141, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-deep-security-review", "bytes": 8141, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T00:11:01.052Z] [trial] id=craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v scenario=craftbook-deep-security-review model=underdog-saluki-27b-iq2-mix", "[2026-11-06T00:11:01.052Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-deep-security-review-IyouBJ", "[2026-11-06T00:11:01.052Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T00:11:01.052Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "[2026-11-06T00:11:01.052Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T00:11:01.055Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T00:11:01.060Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T00:11:01.066Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-deep-security-review-IyouBJ", "[2026-11-06T00:11:01.066Z] [trial] throughput-scaled ceiling: 45m → 99m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-06T00:11:06.681Z] [trial] daemon spawned pid=16924 port=https://127.0.0.1:50599", "[2026-11-06T00:11:08.067Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T00:11:08.082Z] [trial] meester=vanya", "[2026-11-06T00:11:08.082Z] [trial] running scenario setup hook", "[2026-11-06T00:11:08.561Z] [trial] scenario setup complete", "[2026-11-06T00:11:08.563Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T00:11:08.638Z] [poll] initial digests hard=002b0622951c3137 soft=61b1cd1120b52b3f", "[2026-11-06T00:11:08.862Z] [poll] hard-progress: 002b0622951c3137 → 859929c4054d368b after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-06T00:11:49.518Z] [poll] hard-progress: 859929c4054d368b → 24223a147c36547f after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=39,stream=84) sniff=0/0B)", "[2026-11-06T00:12:01.328Z] [poll] hard-progress: 24223a147c36547f → 25aa957ac939ceea after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=59,stream=141) sniff=0/0B)", "[2026-11-06T00:13:23.950Z] [poll] hard-progress: 25aa957ac939ceea → 77c1430c3cba6122 after 83s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=6,slot=284,stream=807) sniff=0/0B)", "[2026-11-06T00:19:50.741Z] [poll] hard-progress: 77c1430c3cba6122 → 0ea414007a985546 after 387s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=1422,stream=4156) sniff=0/0B)", "[2026-11-06T00:20:32.343Z] [poll] hard-progress: 0ea414007a985546 → 6c539d7416abf32f after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=12,slot=1477,stream=4311) sniff=0/0B)", "[2026-11-06T00:22:31.645Z] [poll] retry-loop nudge deferred: rae/47d539af in security-maintenance-eval for 683s, last tool call 161s ago (+1 more) is already working; refusing to queue a potentially stale edit", "[2026-11-06T00:23:19.231Z] [poll] hard-progress: 6c539d7416abf32f → ffd4f15343cf2388 after 167s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=13,slot=1957,stream=6823) sniff=2/0B)", "[2026-11-06T00:26:18.600Z] [poll] hard-progress: ffd4f15343cf2388 → e53b72fc1e90ebe1 after 179s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=14,slot=2451,stream=8647) sniff=4/0B)" ] }, "metrics": { "process": { "sampleCount": 335, "peakRssMb": 15044, "peakCpuPercent": 1232.3, "firstAtMs": 0, "lastAtMs": 1988658 }, "gpu": { "available": true, "sampleCount": 335, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 335, "peakUsedMb": 31275, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 56370, "totalOutputTokens": 16019, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 36168, "todayTokensCached": 22546, "todayTokensOut": 3523, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 36168, "totalTokensCached": 22546, "totalTokensOut": 3523, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11041, "cachedInputTokens": 9456, "outputTokens": 755, "cost": 0, "durationMs": 121430, "outputTokensPerSec": 9.056509206241891, "at": "2026-11-06T00:44:16.382Z" }, "lastUpdated": "2026-11-06T00:44:16.382Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T00:44:17.377Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1990189, "meanTokensPerSec": 8.96, "genTokensPerSec": 8.96, "genTokens": 16019, "promptTokensPerSec": 402.85, "peakRssMb": 15044, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 36168, "output": 3523 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T00:11:01.048Z", "finishedAt": "2026-11-06T00:44:26.601Z", "timeToFirstArtifactMs": 736330, "timeToLastArtifactWriteMs": 1995314, "timeToFirstTokenMs": 31488, "firstTurnTtftMs": 21173, "timeToFirstToolCallMs": 47708 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T00:11:06.739Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T00:11:06.632Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "lindiwe": 1, "rae": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 7, "maxContextFill": 0.1703338623046875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1703338623046875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "scenarioId": "craftbook-deep-security-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-06T01:06:48.717Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/security-review/REPORT.md", "artifactBytes": 8169, "artifactSha256": "4297875621ea5896c67b58dfede56cb5747ec79c467efdff84b7bd80ebe47d94", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/de/deep-security-review/versions/1.1.5/test.json", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ], "scoreAxes": { "verification": 7, "systemic analysis": 6, "remediation": 7 }, "meanScore": 6.67, "justification": "The report and JSON agree on all three real admin flaws and clear the documented safe constructs. Source-to-sink traces and fixes are concrete. Systemic sections are repetitive and overstate unauthenticated exposure and database blast-radius certainty; the shell-allowlist fallback is weaker than the stated no-request-data policy. Middleware limits are acknowledged.", "reviewBasis": "Read all8,169bytes of the final report and the complete2,503-byte findings array against all3 source modules and docs/security.md. Inspected every write, failed script call and standard feedback repair. Compared source files with seeded fixtures; unchanged. Three records agree with the report. The model read the supplied grading oracle; no independent exploit execution is claimed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/security-review/REPORT.md", "bytes": 8169, "sha256": "4297875621ea5896c67b58dfede56cb5747ec79c467efdff84b7bd80ebe47d94", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/security-review/findings.json", "bytes": 2503, "sha256": "7d99283f155060a12be807b165ef9366e3249d31c63f4674f6266932ebac56b4", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/admin.ts", "bytes": 378, "sha256": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/profile.ts", "bytes": 223, "sha256": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/reports.ts", "bytes": 613, "sha256": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/docs/security.md", "bytes": 648, "sha256": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/tests/security-oracle.mjs", "bytes": 4331, "sha256": "e1736cd4940bfe55bac7900c7175d3202343ceec38f6d9a7da17e5e398dba6c7", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:48.633Z", "reviewId": "073", "cellId": "r2-saluki-cuda-developer-craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "scenarioId": "craftbook-deep-security-review", "status": "scored", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "baselineCellId": "r2-saluki-cuda-developer-craftbook-deep-security-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "07e1720e8da315205b1709a066c7090989c1a4cd5dc56e3c26b2ddd5cc2ef5a5", "evidenceSha256": "05b12393970dd6f183d6a7a1df3b37214462fb4169edfa1a510734ec540ffc2a", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/security-review/REPORT.md", "bytes": 8169, "sha256": "4297875621ea5896c67b58dfede56cb5747ec79c467efdff84b7bd80ebe47d94", "shownBytes": 8169, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/security-review/findings.json", "bytes": 2503, "sha256": "7d99283f155060a12be807b165ef9366e3249d31c63f4674f6266932ebac56b4", "shownBytes": 2503, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/admin.ts", "bytes": 378, "sha256": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7", "shownBytes": 378, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/profile.ts", "bytes": 223, "sha256": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29", "shownBytes": 223, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/reports.ts", "bytes": 613, "sha256": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790", "shownBytes": 613, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/docs/security.md", "bytes": 648, "sha256": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d", "shownBytes": 648, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/tests/security-oracle.mjs", "bytes": 4331, "sha256": "e1736cd4940bfe55bac7900c7175d3202343ceec38f6d9a7da17e5e398dba6c7", "shownBytes": 4331, "truncated": false } ], "contextFiles": [], "scoreAxes": { "verification": 8, "systemic analysis": 6, "remediation": 7 }, "justification": "All three seeded defects are correctly confirmed with accurate lines, and the three safe look-alikes are correctly cleared by citing docs/security.md. Themes are grouped as injection vs authorization, with blast radius. Fixes are concrete (parameterized query, archiver or allowlist, requireAdmin). The JSON lacks nothing required.", "limitations": "The report repeats the verified-safe table twice, and the JSON has extra fields (id, description). Systemic analysis is thin, with a stretched \"trust at boundary\" root cause. The F2 high rating and F3's amplification claim are slightly inconsistent. No middleware review was possible.", "meanScore": 7, "durationMs": 14931, "costUsd": 0.0416708, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 9519, "cache_read_input_tokens": 808, "output_tokens": 351, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 9519, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 351, "cache_read_input_tokens": 808, "cache_creation_input_tokens": 9519, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9519 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 351, "cacheReadInputTokens": 808, "cacheCreationInputTokens": 9519, "webSearchRequests": 0, "costUSD": 0.0416708, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 9519, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-deep-security-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 33m 26s; budget used: 33.8%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 736330`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":8141,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.33797281661511686`; `facts.toolUse.totalToolCalls: 28`; `facts.timing.timeToFirstArtifactMs: 736330`; `facts.outcome.durationMs: 2005553` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 5 + 0.15 * 10 = **9.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15044 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31275 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.96 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 56370 + 16019 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 17% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 736330`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":8141,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.33797281661511686`\n- `facts.toolUse.totalToolCalls: 28`\n- `facts.timing.timeToFirstArtifactMs: 736330`\n- `facts.outcome.durationMs: 2005553`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 736330`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":8141,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.33797281661511686`; `facts.toolUse.totalToolCalls: 28`; `facts.timing.timeToFirstArtifactMs: 736330`; `facts.outcome.durationMs: 2005553`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS9/10** is unchanged. Advisory **6.67/10**: verification7, systemic analysis6, remediation7; codex-inline/current-session.\n\nThe report and JSON agree on all three real admin flaws and clear the documented safe constructs. Source-to-sink traces and fixes are concrete. Systemic sections are repetitive and overstate unauthenticated exposure and database blast-radius certainty; the shell-allowlist fallback is weaker than the stated no-request-data policy. Middleware limits are acknowledged.\n\nRead all8,169bytes of the final report and the complete2,503-byte findings array against all3 source modules and docs/security.md. Inspected every write, failed script call and standard feedback repair. Compared source files with seeded fixtures; unchanged. Three records agree with the report. The model read the supplied grading oracle; no independent exploit execution is claimed.\n\n## Positives\n\n- All11 fixed checks pass, including the harness-owned precision/recall script. The final JSON array contains exactly3 real admin findings: shell injection at line4, SQL injection at line7 and missing requireAdmin at line3. Report and JSON agree on severities and exclude the documented safe patterns (outcome.success; sniff.latest; artifacts.otherFileCount; advisory-consistency-audit.json).\n- The report traces both untrusted query parameters to their actual sinks and gives concrete in-process archive, placeholder-query and middleware fixes. It correctly separates the fixed logrotate command, mapped identifier, authenticated parameterized profile query and intentionally public health response (toolUse.byTool.read_file=14; final report/findings versus seeded modules and policy).\n- All four source/policy files remain byte-equivalent after newline normalization. The final report adds a limitation that middleware implementations are absent and need follow-up review (artifacts.otherFileCount; sourceComparisons in advisory-consistency-audit.json; final Not Statically Verifiable section).\n\n## Negatives\n\n- The report repeatedly treats local missing middleware as proof of unauthenticated deployed reachability. The reviewed fixture establishes a missing mandated per-route boundary, but does not expose the entire deployment stack. Likewise, saying SQL impact is bounded to users is not supported without database permissions/driver details; the report elsewhere acknowledges driver-dependent effects (artifacts.otherFileCount; final report versus supplied modules/policy).\n- The preferred in-process archive fix follows the policy; the fallback of putting allowlisted request data into a shell command is weaker than docs/security.md’s explicit no-request-data shell rule. Validation is not a substitute for removing unsafe shell interpolation or SQL parameterization (artifacts.otherFileCount; final report/findings remediation versus docs/security.md).\n- Two actors alternate writes to the same deliverables. The first findings write wraps records in an object and fails the required top-level array shape; a second actor corrects it, followed by another overwrite. The report needs multiple repairs and ends with two duplicated safe-construct tables (toolUse.byTool.write_file=7, replace_in_file=1; tool receipts; log.txt; advisory-consistency-audit.json).\n- The first report draft is refused because its proposed400 rejection code is absent from source evidence. Later repairs remove that specific value and add required systemic headings. The write guard is conflating a remediation proposal with an asserted source fact (toolUse.byTool.write_file; rejected tool receipt00:30:36Z).\n- The model reads tests/security-oracle.mjs at00:12:01Z before writing and later calls run_installed_script(security-oracle), which fails because no project script is installed. Model validation checks only JSON parsing. The standard harness separately executes its oracle successfully; these are distinct verification paths (toolUse.byTool.run_installed_script=1, validate=1, read_file=14; sniff.latest; advisory-tool-evidence.json).\n- No scenario-scoped craftbook task/stage execution is captured: the only continuity steps belong to default/shared and none complete. This pass demonstrates deliverable production under the adapter, not five-stage craftbook execution (continuity.steps.completed=0; continuity.sessions.taskScoped=0; toolUse.byTool has no invoke_craftbook or advance_task_step).\n\n## Performance and GPU fit\n\nElapsed **33m26s**,28calls,16,019 engine-generated tokens at **8.96tok/s**, prefill402.85tok/s (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10,640/12,227MiB**, process-tree RSS **15,044MiB**, whole-system RAM **31,275/31,865MiB** (perf.gpu; perf.process; perf.systemMemory). Whole-system RAM includes other applications; sharedGPU memory and paging were not measured.\n\nOne native0.1.57CUDA launch,65,536context,one slot,f16KV,reasoning4096. DenseFFN39/64GPU and25CPU: hybrid residency. Maximum captured session fill17.0%; largest observed released native slot14872tokens (22.7%). No compaction, endpoint failure, explicit OOM or context overflow. These sampled context observations are not continuous KV peaks. No new campaign hardware peak.\n\n## Tactical experiment — not applied\n\nIn Gilde deep-security-review craftbook.json based on1.1.5, report/evaluate prompts should require: “Separate confirmed local dataflow, deployment assumptions and driver-dependent impact. Preserve the project policy in every proposed fallback. Keep cleared constructs in one section and distinguish model-run checks from harness feedback.” Measure overclaimed findings and report/JSON consistency without encoding the seeded flaws.\n\n## Strategic experiments — not applied\n\nIn packages/service/src/chat/manager.ts and the artifact write path, test one authoritative writer or content-version conflict checks during automatic specialist repair. Preserve the latest accepted findings while another worker reviews them; measure lost sections, duplicate sections and total rewrites. In packages/core/src/checks/claims.ts, classify proposed remediation values separately from assertions about source facts.\n\nSeparate the script-file execution surface from run_installed_script project registrations; the error already explains this distinction, but the model still chooses the wrong surface. Preserve harness-only oracle isolation in future eval revisions and label direct adapter delivery separately from actual craftbook-stage execution. Do not change any of these boundaries mid-campaign.\n\n## Generalization and next experiment\n\nR1 fixed5.3/advisory4 failed after the report/JSON diverged and a second worker weakened the output. R2 fixed9/advisory6.67 produces coherent final findings despite alternating writes. This is improved repetition outcome under the same frozen native/tuning identity, not evidence of a native-version change.\n\nTest write coordination, proposal/fact separation and verification provenance across this task, unrelated A/B reporting and API review. Measure consistency, supported impact claims, false write rejections and completion cost; the same oracle-exposure caveat appears in API review.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/facts.json): fail, fixed5.3, 19.52min, 8.95tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/facts.json): fail, fixed6.3, 3.20min, 132.51tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-29-14-510Z-r1ri/facts.json): fail, fixed6.3, 2.92min, 131.6tok/s.\n\nOther-model history is contextual, not a controlled native-version comparison.\n\n## Strategic note\n\nThe output is materially more useful than R1 and passed the standard oracle. Preserve the remaining scope/impact and evaluation-provenance limits. No exploit was executed, no model retry was launched, and no source, gate, tuning or campaign state changed.\n\n## Raw evidence\n\nfacts.json,score.json,advisory-artifact-review.json,advisory-consistency-audit.json,advisory-tool-evidence.json,historical-comparison.json and captured source/deliverables.\n" }, { "cellId": "r2-saluki-cuda-developer-large-pr-review", "baselineCellId": "r2-saluki-cuda-developer-large-pr-review", "suite": "developer", "scenario": "large-pr-review", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T00:47:11.238Z", "finishedAt": "2026-11-06T02:28:49.704Z", "durationMs": 6098466, "success": true, "reason": "complete large-PR review (coverage-120-of-120, coverage-summary, late-defect-citation, request-changes, no-false-missing-api, source-unchanged)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 4572076 }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.770778333102463 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 45 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 4572076 }, { "source": "facts", "path": "outcome.durationMs", "value": 6098466 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 1028, "peakRssMb": 14829, "peakCpuPercent": 1216.8, "firstAtMs": 0, "lastAtMs": 6081926 }, "gpu": { "available": true, "sampleCount": 1028, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1028, "peakUsedMb": 31050, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 375238, "totalOutputTokens": 35627, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 43652, "todayTokensCached": 43570, "todayTokensOut": 735, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 43652, "totalTokensCached": 43570, "totalTokensOut": 735, "totalCost": 0, "medianOutputTokensPerSec": 8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 43652, "cachedInputTokens": 43570, "outputTokens": 735, "cost": 0, "durationMs": 4956457, "outputTokensPerSec": 7.969860982605246, "at": "2026-11-06T02:09:58.716Z" }, "lastUpdated": "2026-11-06T02:09:58.716Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T02:28:40.805Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6083448, "meanTokensPerSec": 8.02, "genTokensPerSec": 8.02, "genTokens": 35627, "promptTokensPerSec": 303.23, "peakRssMb": 14829, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43652, "output": 735 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T00:47:17.013Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T00:47:16.909Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "rina": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.457275390625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.457275390625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "complete large-PR review (coverage-120-of-120, coverage-summary, late-defect-citation, request-changes, no-false-missing-api, source-unchanged)", "durationMs": 6098466, "timeoutMs": 7912088, "budgetUsedFraction": 0.770778333102463 }, "timing": { "startedAt": "2026-11-06T00:47:11.238Z", "finishedAt": "2026-11-06T02:28:49.704Z", "timeToFirstArtifactMs": 4572076, "timeToLastArtifactWriteMs": 6085472, "timeToFirstTokenMs": 39971, "firstTurnTtftMs": 28951, "timeToFirstToolCallMs": 50002 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 45, "byTool": { "list_artifacts": 3, "list_dir": 1, "read_artifact": 9, "read_file": 5, "write_artifact": 2, "read_artifacts": 17, "replace_in_file": 2, "write_file": 3, "validate": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 130 }, "sniff": { "progression": [], "latest": null }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T00:47:11.241Z] [trial] id=large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s scenario=large-pr-review model=underdog-saluki-27b-iq2-mix", "[2026-11-06T00:47:11.241Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T00:47:11.241Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "[2026-11-06T00:47:11.241Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-large-pr-review-6rsLYy", "[2026-11-06T00:47:11.242Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T00:47:11.244Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T00:47:11.250Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T00:47:11.257Z] [trial] throughput-scaled ceiling: 60m → 132m at 9.1 tok/s (reference 20 tok/s)", "[2026-11-06T00:47:11.257Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-large-pr-review-6rsLYy", "[2026-11-06T00:47:16.963Z] [trial] daemon spawned pid=40564 port=https://127.0.0.1:52237", "[2026-11-06T00:47:18.396Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T00:47:18.413Z] [trial] meester=cassian", "[2026-11-06T00:47:18.414Z] [trial] running scenario setup hook", "[2026-11-06T00:47:19.791Z] [trial] scenario setup complete", "[2026-11-06T00:47:19.793Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T00:47:19.856Z] [poll] initial digests hard=41fa20d7a3e10dcb soft=d4f468f83c773cec", "[2026-11-06T00:47:19.949Z] [poll] hard-progress: 41fa20d7a3e10dcb → 64970e927dab6962 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-06T00:48:05.714Z] [poll] hard-progress: 64970e927dab6962 → 4af8b4d93f6d79e4 after 46s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=32,stream=63) sniff=0/0B)", "[2026-11-06T00:48:52.920Z] [poll] hard-progress: 4af8b4d93f6d79e4 → e8841013c30ebe52 after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=90,stream=263) sniff=0/0B)", "[2026-11-06T00:51:20.768Z] [poll] hard-progress: e8841013c30ebe52 → 8527b96930a8ae6f after 148s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=346,stream=1009) sniff=0/0B)", "[2026-11-06T00:53:47.714Z] [poll] hard-progress: 8527b96930a8ae6f → 7d6fd09c63840a80 after 147s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=715,stream=2400) sniff=0/0B)", "[2026-11-06T00:54:58.398Z] [poll] hard-progress: 7d6fd09c63840a80 → 64af9dccca80fb45 after 71s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=855,stream=3101) sniff=0/0B)", "[2026-11-06T00:56:21.454Z] [poll] hard-progress: 64af9dccca80fb45 → c128591875cc72d1 after 83s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=12,slot=991,stream=3788) sniff=0/0B)", "[2026-11-06T00:56:21.457Z] [poll] retry-loop nudge deferred: rina/0e29a4a3 in large-pr-review-eval for 542s, last tool call 2s ago is already working; refusing to queue a potentially stale edit", "[2026-11-06T00:57:50.405Z] [poll] hard-progress: c128591875cc72d1 → 134a208307309ab8 after 89s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=13,slot=1034,stream=3903) sniff=0/0B)" ] }, "metrics": { "process": { "sampleCount": 1028, "peakRssMb": 14829, "peakCpuPercent": 1216.8, "firstAtMs": 0, "lastAtMs": 6081926 }, "gpu": { "available": true, "sampleCount": 1028, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1028, "peakUsedMb": 31050, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 375238, "totalOutputTokens": 35627, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 43652, "todayTokensCached": 43570, "todayTokensOut": 735, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 43652, "totalTokensCached": 43570, "totalTokensOut": 735, "totalCost": 0, "medianOutputTokensPerSec": 8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 43652, "cachedInputTokens": 43570, "outputTokens": 735, "cost": 0, "durationMs": 4956457, "outputTokensPerSec": 7.969860982605246, "at": "2026-11-06T02:09:58.716Z" }, "lastUpdated": "2026-11-06T02:09:58.716Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T02:28:40.805Z" } }, "source": "engine-log" }, "derived": { "durationMs": 6083448, "meanTokensPerSec": 8.02, "genTokensPerSec": 8.02, "genTokens": 35627, "promptTokensPerSec": 303.23, "peakRssMb": 14829, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 43652, "output": 735 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T00:47:11.238Z", "finishedAt": "2026-11-06T02:28:49.704Z", "timeToFirstArtifactMs": 4572076, "timeToLastArtifactWriteMs": 6085472, "timeToFirstTokenMs": 39971, "firstTurnTtftMs": 28951, "timeToFirstToolCallMs": 50002 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T00:47:17.013Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T00:47:16.909Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "rina": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.457275390625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.457275390625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-large-pr-review", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "scenarioId": "large-pr-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-06T03:10:41.976Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/workspace/large-pr-review-eval/pr-review.md", "artifactBytes": 1799, "artifactSha256": "a93d7c9bd0c9adf291b11310cbb11198955eb46a3a0558e0e985260fe708cb51", "axisSource": "evals/src/scenarios/large-pr-review.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared qualitative axes. The final report correctly identifies the authorization flaw, clears the valid API and cites line4. All120 records have successful complete read receipts and the final workspace ledger matches them. The fixture names the defect in advance; delayed bulk ledger writes and a stale alternate copy weaken the process evidence.", "reviewBasis": "Read final report and complete final ledger, compared all paths with the corpus, inspected the final repair receipts and retained prior per-item read audit. Read relevant source modules. This is a coverage and synthesis task with an explicitly disclosed flaw, not blind vulnerability discovery.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/workspace/large-pr-review-eval/pr-review.md", "status": "reviewedNoAxes", "bytes": 1799, "sha256": "a93d7c9bd0c9adf291b11310cbb11198955eb46a3a0558e0e985260fe708cb51" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/workspace/large-pr-review-eval/pr-review-coverage.json", "status": "reviewedSupportingArtifact", "bytes": 14229 } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:51.406Z", "reviewId": "074", "cellId": "r2-saluki-cuda-developer-large-pr-review", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "scenarioId": "large-pr-review", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "baselineCellId": "r2-saluki-cuda-developer-large-pr-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "05d51ebeba3f4837175fa7316cbf70f40f6836031e444a87fc0983e072619b9b", "evidenceSha256": "33e366e34f7d7f0d7ae33d57d47c1a28681f0be8f455dfee9b02eecd4a248d40", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/workspace/large-pr-review-eval/pr-review.md", "bytes": 1799, "sha256": "a93d7c9bd0c9adf291b11310cbb11198955eb46a3a0558e0e985260fe708cb51", "shownBytes": 1799, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/workspace/large-pr-review-eval/pr-review-coverage.json", "bytes": 14229, "sha256": "d67e7e39a91d561d7cc069143cff8f1f6bb91a7a21a5dc1c09af03c617157b3d", "shownBytes": 14229, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. Report has \"Coverage: 120/120\", a findings table, \"Verdict: request-changes\". It cites the auth defect at src/security/late-authorization.ts:4. It treats assuredApi as defined later, not missing. The coverage JSON lists 120 files and 120 records, ending with files 119 and 120.", "limitations": "Batch sizes (≤25), tool verification via find_symbol/grep, and no source modification can't be confirmed from the artifacts alone. Line number 4 and the cited sources weren't checked against the patch records. Observations notes weren't shown.", "meanScore": null, "durationMs": 14727, "costUsd": 0.043075999999999996, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 9963, "cache_read_input_tokens": 700, "output_tokens": 315, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 9963, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 315, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 9963, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9963 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 315, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 9963, "webSearchRequests": 0, "costUSD": 0.043075999999999996, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 9963, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - large-pr-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 101m 38s; budget used: 77.1%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 4572076`; `facts.sniff.latest: null` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.770778333102463`; `facts.toolUse.totalToolCalls: 45`; `facts.timing.timeToFirstArtifactMs: 4572076`; `facts.outcome.durationMs: 6098466` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14829 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31050 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.02 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 375238 + 35627 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 46% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 4572076`\n- `facts.sniff.latest: null`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.770778333102463`\n- `facts.toolUse.totalToolCalls: 45`\n- `facts.timing.timeToFirstArtifactMs: 4572076`\n- `facts.outcome.durationMs: 6098466`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 4572076`; `facts.sniff.latest: null`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.770778333102463`; `facts.toolUse.totalToolCalls: 45`; `facts.timing.timeToFirstArtifactMs: 4572076`; `facts.outcome.durationMs: 6098466`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.5/10** remains unchanged. No declared qualitative axes; no advisory score invented.\n\nNo declared qualitative axes. The final report correctly identifies the authorization flaw, clears the valid API and cites line4. All120 records have successful complete read receipts and the final workspace ledger matches them. The fixture names the defect in advance; delayed bulk ledger writes and a stale alternate copy weaken the process evidence.\n\nRead final report and complete final ledger, compared all paths with the corpus, inspected the final repair receipts and retained prior per-item read audit. Read relevant source modules. This is a coverage and synthesis task with an explicitly disclosed flaw, not blind vulnerability discovery.\n\n## Positives\n\n- All fixed conditions pass: 120 exact changed paths, coverage summary, precise late-defect citation, request-changes verdict, valid API clearance and unchanged sources (outcome.success; outcome.reason; final report and advisory-consistency-audit.json).\n- Unlike R1, all120 patch records have actual per-item OK/complete receipts, not merely a completed coverage ledger. The API definition and authorization module were also read from the checkout at00:51:15Z (toolUse.byTool.read_artifacts=17; toolUse.byTool.read_file=5; saved per-item receipts).\n- The report describes the cross-owner deletion path correctly and gives a direct ownership-comparison fix without editing the source (artifacts.otherFileCount; outcome.reason includes source-unchanged; final report).\n\n## Negatives\n\n- The mission explicitly names the late authorization defect and warns that the API exists. Correct citation and synthesis do not demonstrate independent discovery in an unknown codebase (outcome.reason; scenario setup mission; toolUse.byTool.read_file).\n- The prompt requests a ledger update after each batch, but the empty scaffold persists until a bulk artifact write; a later workspace write corrects six filenames and a final rewrite adds full paths. The alternate artifact ledger remains stale (toolUse.byTool.write_artifact=2, write_file=3, replace_in_file=2; saved ledgers and tool receipts).\n- Fourteen guessed record paths fail before corrected reads. read_file silently reroutes a workspace request to an artifact, then replace_in_file fails against the absent workspace file. Validation checks the stale artifact ledger for JSON syntax, and the report first in the wrong area (toolUse.byTool.validate=3; toolUse.byTool.replace_in_file=2; per-item errors and tool receipts).\n- The initial range1-4 citation includes the bad return but the fixed regex accepts only line4. Standard feedback is followed about21minutes later by a full ledger rewrite and two citation replacements. This is repair overhead, not a newly discovered flaw. The remaining API reference names the declaration at line3 instead of the call at line4 (outcome.durationMs=6098466; toolUse.totalToolCalls=45; log.txt and final report).\n\n## Performance and GPU fit\n\nElapsed **101m38s**, 45 tool calls, **35627** engine-generated tokens at **8.02tok/s**, prefill 303.23tok/s (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10646/12227MiB**, process-tree RSS **14829MiB**, whole-system RAM **31050/31865MiB** (perf.gpu; perf.process; perf.systemMemory). System RAM includes other applications; shared GPU memory and paging are unmeasured.\n\nOne native0.1.57CUDA launch,65536context,one slot,f16KV. DenseFFN39/64GPU and25CPU: hybrid residency, not full GPU. Maximum captured session fill 45.7%; largest observed released native slot 52630tokens (80.3%). No observed compaction, endpoint failure, explicit OOM or context overflow. Context observations are not continuous allocation peaks. No new campaign memory peak.\n\n## Tactical experiment — not applied\n\nExperiment in the document-writing instruction in packages/core/src/prompt/instructions.ts: add “Use the requested output area explicitly; retain the area returned by a read, and confirm that edits target the same file. Save coverage progress after each completed batch.” Measure wrong-area failures and lost progress across corpus review and unrelated research-to-document tasks (toolUse.byTool.replace_in_file and validate; saved stale ledger). No prompt or sampling change applied.\n\n## Strategic experiments — not applied\n\nIn packages/mcp/src/cross-drawer-read-tools.ts, expose canonical area/path metadata consistently to subsequent edit and validation calls rather than relying only on a prose reroute notice. In evals/src/scenarios/large-pr-review.ts, a future separately versioned evaluator should parse path/line ranges and accept a range that contains the evidenced defect; add read-receipt coverage checks instead of trusting ledger claims. Preserve this campaign’s frozen gate and identity.\n\n## Generalization and next experiment\n\nR1 and R2 both fixed8.5; R1 had only55 observed distinct successful record requests, while R2 delivers all120. No model/native/tuning change occurred between repetitions. Unrelated codemod and API/security reviews show that plausible coverage/verification prose needs independent tool receipts. The large-PR improvement is observed read coverage, not proof of general autonomous defect discovery.\n\nTest explicit tool availability and receipt-backed verification across corpus review, codemod and an unrelated report task. Measure unsupported validation claims, missing required tools, repair latency and task outcomes; leave all current gates/tuning untouched. Scenario-specific line-range acceptance would be an evaluator correction for future identity, not model prompt training.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/facts.json): pass, fixed8.5, 86.70min, 8.27tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/facts.json): pass, fixed8.5, 23.00min, 136.87tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T12-48-58-694Z-mo2i/facts.json): fail, fixed3.3, 22.49min, 134.89tok/s.\n\nOther-model runs are contextual and not a controlled native-version comparison.\n\n## Strategic note\n\nSuccessful loading and fixed gates do not establish all claimed work was executed. Preserve the coverage/provenance distinctions above. No campaign rerun, source edit, model download, tuning change or gate change was made.\n\n## Raw evidence\n\nfacts.json,score.json,advisory-artifact-review.json,advisory-consistency-audit.json,advisory-tool-evidence.json,historical-comparison.json and the captured artifacts/workspace.\n" }, { "cellId": "r2-saluki-cuda-developer-craftbook-codemod-sweep", "baselineCellId": "r2-saluki-cuda-developer-craftbook-codemod-sweep", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T02:32:28.349Z", "finishedAt": "2026-11-06T03:04:24.161Z", "durationMs": 1915812, "success": true, "reason": "craftbook-codemod-sweep passed 32 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 501237 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 32, "scoreMax": 32, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.14173461125986322 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 49 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 501237 }, { "source": "facts", "path": "outcome.durationMs", "value": 1915812 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 321, "peakRssMb": 15671, "peakCpuPercent": 8944.5, "firstAtMs": 0, "lastAtMs": 1903313 }, "gpu": { "available": true, "sampleCount": 321, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 321, "peakUsedMb": 31515, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 59835, "totalOutputTokens": 14564, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 59743, "todayTokensCached": 56596, "todayTokensOut": 2921, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 59743, "totalTokensCached": 56596, "totalTokensOut": 2921, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10669, "cachedInputTokens": 10251, "outputTokens": 256, "cost": 0, "durationMs": 141661, "outputTokensPerSec": 8.696713276729895, "at": "2026-11-06T03:04:14.468Z" }, "lastUpdated": "2026-11-06T03:04:14.468Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T03:04:18.446Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1904920, "meanTokensPerSec": 8.79, "genTokensPerSec": 8.79, "genTokens": 14564, "promptTokensPerSec": 387.18, "peakRssMb": 15671, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 59743, "output": 2921 } } } }, "craftbook": { "craftbookId": "codemod-sweep", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 5, "completed": 5, "gateApprovals": 5, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "profile-cards/1", "stepId": "enumerate", "activatedAt": "2026-11-06T02:32:34.761Z", "completedAt": "2026-11-06T02:40:49.760Z", "ms": 494999, "gateRejections": 0, "toolCalls": 16, "sessionIds": [ "bbeb3170-2b3a-42d7-9270-09eedd0e0e13" ] }, { "ref": "profile-cards/1", "stepId": "apply", "activatedAt": "2026-11-06T02:40:49.762Z", "completedAt": "2026-11-06T02:44:03.621Z", "ms": 193859, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "447d78f4-698d-4812-b2e9-0b7a03748a68" ] }, { "ref": "profile-cards/1", "stepId": "verify", "activatedAt": "2026-11-06T02:44:03.632Z", "completedAt": "2026-11-06T02:52:11.825Z", "ms": 488193, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "447d78f4-698d-4812-b2e9-0b7a03748a68" ] }, { "ref": "profile-cards/1", "stepId": "evaluate", "activatedAt": "2026-11-06T02:59:18.676Z", "completedAt": "2026-11-06T03:01:46.684Z", "ms": 148008, "gateRejections": 1, "toolCalls": 9, "sessionIds": [ "b9fe9c43-6805-44e4-bb41-4c909b997e9e" ] }, { "ref": "profile-cards/1", "stepId": "finish", "activatedAt": "2026-11-06T03:01:46.686Z", "completedAt": "2026-11-06T03:04:14.427Z", "ms": 147741, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "cb263589-fd50-422f-8c6a-226138850e15" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T02:32:33.550Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T02:32:33.462Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 193859 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "aarav": 1, "charles": 1, "vasiliki": 1, "zofia": 1 }, "perTask": { "profile-cards/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.13360595703125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.13360595703125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-codemod-sweep passed 32 deterministic craftbook checks", "durationMs": 1915812, "timeoutMs": 13516896, "budgetUsedFraction": 0.14173461125986322 }, "timing": { "startedAt": "2026-11-06T02:32:28.349Z", "finishedAt": "2026-11-06T03:04:24.161Z", "timeToFirstArtifactMs": 501237, "timeToLastArtifactWriteMs": 1758363, "timeToFirstTokenMs": 35767, "firstTurnTtftMs": 20557, "timeToFirstToolCallMs": 48845 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Codebase Analyst", "Developer", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 49, "byTool": { "read_file": 10, "list_dir": 8, "grep_files": 3, "write_artifact": 6, "read_artifact": 7, "read_files": 3, "replace_in_file": 4, "run_package_script": 2, "read_artifacts": 1, "stat": 1, "advance_task_step": 3, "write_task_note": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 16 }, "sniff": { "progression": [ { "atMs": 6986, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 9, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/api.js, src/index.js, src/profile.js, tests/profile.test.mjs, CHANGELOG.md, source/rename-brief.md" }, { "atMs": 54618, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 9, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/api.js, src/index.js, src/profile.js, tests/profile.test.mjs, CHANGELOG.md" }, { "atMs": 66448, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 9, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: src/api.js, src/index.js, src/profile.js, tests/profile.test.mjs" }, { "atMs": 95984, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 10, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 505137, "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 15, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 611909, "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 20, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 695313, "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 24, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 1184332, "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 27, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 1600382, "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 29, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 1612503, "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 30, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 1880445, "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 31, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 1909779, "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 32, "scoreMax": 32, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-codemod-sweep", "bytes": 2681, "score": 32, "scoreMax": 32, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T02:32:28.352Z] [trial] id=craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z scenario=craftbook-codemod-sweep model=underdog-saluki-27b-iq2-mix", "[2026-11-06T02:32:28.353Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "[2026-11-06T02:32:28.353Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-codemod-sweep-LfGmU8", "[2026-11-06T02:32:28.353Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T02:32:28.353Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T02:32:28.357Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T02:32:28.361Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T02:32:28.366Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-codemod-sweep-LfGmU8", "[2026-11-06T02:32:28.366Z] [trial] throughput-scaled ceiling: 90m → 225m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T02:32:33.488Z] [trial] daemon spawned pid=21928 port=https://127.0.0.1:52440", "[2026-11-06T02:32:34.375Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-06T02:32:34.391Z] [trial] meester=aldric", "[2026-11-06T02:32:34.391Z] [trial] running scenario setup hook", "[2026-11-06T02:32:34.916Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T02:32:34.915Z] [trial] scenario setup complete", "[2026-11-06T02:32:34.977Z] [poll] initial digests hard=ac38ceee5938e3d9 soft=5545bef77f6b5acc", "[2026-11-06T02:32:35.383Z] [poll] hard-progress: ac38ceee5938e3d9 → 4bc98524240a2174 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-06T02:32:41.692Z] [poll] hard-progress: 4bc98524240a2174 → ea02700d6690cabe after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-06T02:33:23.037Z] [poll] hard-progress: ea02700d6690cabe → 6027e2a5d04aef10 after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=49,stream=118) sniff=9/0B)", "[2026-11-06T02:33:34.859Z] [poll] hard-progress: 6027e2a5d04aef10 → 5a22921ececaddef after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=70,stream=175) sniff=9/0B)", "[2026-11-06T02:34:04.400Z] [poll] hard-progress: 5a22921ececaddef → 88b21f7c8d6fefbe after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=12,slot=121,stream=325) sniff=10/0B)", "[2026-11-06T02:35:56.348Z] [poll] hard-progress: 88b21f7c8d6fefbe → 20d21aa84a7018b7 after 112s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=14,slot=440,stream=1278) sniff=10/0B)", "[2026-11-06T02:39:06.324Z] [poll] hard-progress: 20d21aa84a7018b7 → c26b4b09e76408ed after 190s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=15,slot=970,stream=3419) sniff=10/0B)", "[2026-11-06T02:40:53.526Z] [poll] hard-progress: c26b4b09e76408ed → abaa14504d88a5f6 after 107s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=16,slot=1259,stream=4945) sniff=15/2681B)", "[2026-11-06T02:40:59.663Z] [poll] hard-progress: abaa14504d88a5f6 → 6533031f6ad58247 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=16,slot=1259,stream=4945) sniff=15/2681B)" ] }, "metrics": { "process": { "sampleCount": 321, "peakRssMb": 15671, "peakCpuPercent": 8944.5, "firstAtMs": 0, "lastAtMs": 1903313 }, "gpu": { "available": true, "sampleCount": 321, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 321, "peakUsedMb": 31515, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 59835, "totalOutputTokens": 14564, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 59743, "todayTokensCached": 56596, "todayTokensOut": 2921, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 59743, "totalTokensCached": 56596, "totalTokensOut": 2921, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 10669, "cachedInputTokens": 10251, "outputTokens": 256, "cost": 0, "durationMs": 141661, "outputTokensPerSec": 8.696713276729895, "at": "2026-11-06T03:04:14.468Z" }, "lastUpdated": "2026-11-06T03:04:14.468Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T03:04:18.446Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1904920, "meanTokensPerSec": 8.79, "genTokensPerSec": 8.79, "genTokens": 14564, "promptTokensPerSec": 387.18, "peakRssMb": 15671, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 59743, "output": 2921 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T02:32:28.349Z", "finishedAt": "2026-11-06T03:04:24.161Z", "timeToFirstArtifactMs": 501237, "timeToLastArtifactWriteMs": 1758363, "timeToFirstTokenMs": 35767, "firstTurnTtftMs": 20557, "timeToFirstToolCallMs": 48845 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 5, "completed": 5, "gateApprovals": 5, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "profile-cards/1", "stepId": "enumerate", "activatedAt": "2026-11-06T02:32:34.761Z", "completedAt": "2026-11-06T02:40:49.760Z", "ms": 494999, "gateRejections": 0, "toolCalls": 16, "sessionIds": [ "bbeb3170-2b3a-42d7-9270-09eedd0e0e13" ] }, { "ref": "profile-cards/1", "stepId": "apply", "activatedAt": "2026-11-06T02:40:49.762Z", "completedAt": "2026-11-06T02:44:03.621Z", "ms": 193859, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "447d78f4-698d-4812-b2e9-0b7a03748a68" ] }, { "ref": "profile-cards/1", "stepId": "verify", "activatedAt": "2026-11-06T02:44:03.632Z", "completedAt": "2026-11-06T02:52:11.825Z", "ms": 488193, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "447d78f4-698d-4812-b2e9-0b7a03748a68" ] }, { "ref": "profile-cards/1", "stepId": "evaluate", "activatedAt": "2026-11-06T02:59:18.676Z", "completedAt": "2026-11-06T03:01:46.684Z", "ms": 148008, "gateRejections": 1, "toolCalls": 9, "sessionIds": [ "b9fe9c43-6805-44e4-bb41-4c909b997e9e" ] }, { "ref": "profile-cards/1", "stepId": "finish", "activatedAt": "2026-11-06T03:01:46.686Z", "completedAt": "2026-11-06T03:04:14.427Z", "ms": 147741, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "cb263589-fd50-422f-8c6a-226138850e15" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T02:32:33.550Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T02:32:33.462Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 193859 }, "sessions": { "total": 4, "taskScoped": 4, "byGezel": { "aarav": 1, "charles": 1, "vasiliki": 1, "zofia": 1 }, "perTask": { "profile-cards/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.13360595703125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.13360595703125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "scenarioId": "craftbook-codemod-sweep", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-06T03:10:42.087Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/verification.md", "artifactBytes": 2895, "artifactSha256": "ec138f2348265a86691f29c8a26e17a3bbbaf146bbd003c56aa6cbb1b34d2674", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/co/codemod-sweep/versions/1.0.6/test.json", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ], "scoreAxes": { "Enumeration completeness": 8, "Invariant discipline": 9, "Reconciled verification": 8 }, "axisJustifications": { "Enumeration completeness": "A real pre-edit ripgrep search identifies all11sites, including re-export and log text; the7edits and4documented exclusions match the fixture.", "Invariant discipline": "All4changed files equal the literal requested rename, with all4protected inputs byte-identical. No source behavior drift is observed.", "Reconciled verification": "Counts reconcile and both real test runs pass3/3. Manual file inspection and the separate harness oracle support zero residuals; the claimed fresh search is not recorded because grep_files was filtered out during verification." }, "meanScore": 8.33, "justification": "Real pre-edit enumeration covers7edits and4exclusions. All changed source files contain only the rename; protected fixtures are unchanged. Counts reconcile and both test runs pass3/3. Verification checks files manually but presents a fresh search without a receipt; its filtered tool surface omitted grep_files. One review-format rejection was repaired.", "reviewBasis": "Read the complete2895-byte declared verification artifact and three supporting reports, all source/test fixtures and brief. Compared each non-harness fixture byte-for-byte or against the literal rename. Inspected every search, edit and test call, plus model-facing tool surfaces and gate repair. No extra trial or test was run.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/verification.md", "bytes": 2895, "sha256": "ec138f2348265a86691f29c8a26e17a3bbbaf146bbd003c56aa6cbb1b34d2674", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/sites.md", "bytes": 2709, "sha256": "194616bcfd95e2d2047c141b4706d4debe31e03c6b98614f0dc081860e850558", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/change-notes.md", "bytes": 1897, "sha256": "27e1914ebea58444385d008157ca2769b9b66220f8d633492bf9f8ad9ac860ce", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/review.md", "bytes": 3418, "sha256": "13b7813f1f0cbd6419cd56743b99c1592fcf686237a86eae32d2c2943c68fdeb", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:02:55.049Z", "reviewId": "075", "cellId": "r2-saluki-cuda-developer-craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "scenarioId": "craftbook-codemod-sweep", "status": "scored", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "baselineCellId": "r2-saluki-cuda-developer-craftbook-codemod-sweep", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "859033c16cfb1bbd7776e18eaf273e691da80dbab5bd579042878fa97de0bf13", "evidenceSha256": "7a04b4043aa277e38ad2ca5b5b2e17accb0eb28c2a33005ca86a8769ac1ad8c7", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/verification.md", "bytes": 2895, "sha256": "ec138f2348265a86691f29c8a26e17a3bbbaf146bbd003c56aa6cbb1b34d2674", "shownBytes": 2895, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/sites.md", "bytes": 2709, "sha256": "194616bcfd95e2d2047c141b4706d4debe31e03c6b98614f0dc081860e850558", "shownBytes": 2709, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/change-notes.md", "bytes": 1897, "sha256": "27e1914ebea58444385d008157ca2769b9b66220f8d633492bf9f8ad9ac860ce", "shownBytes": 1897, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/review.md", "bytes": 3418, "sha256": "13b7813f1f0cbd6419cd56743b99c1592fcf686237a86eae32d2c2943c68fdeb", "shownBytes": 3418, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z\\workspace\\profile-cards\\source\\rename-brief.md", "bytes": 534, "sha256": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "shownBytes": 534, "truncated": false } ], "scoreAxes": { "Enumeration completeness": 8, "Invariant discipline": 8, "Reconciled verification": 8 }, "justification": "sites.md lists all 11 sites (definition, re-export, 2 imports, 2 calls, log string) with file:line. It also excludes the changelog and brief with reasons tied to the brief. The counts reconcile (11 = 7 + 4). The residual scan is quoted, and the npm test output shows 3/3 passing. Notes claim only identifiers changed.", "limitations": "Evidence is artifact text only. I can't confirm the pre-edit search or the test run happened, or that excluded files are unmodified. No before/after diff is shown. Invariant discipline rests on the authors' and reviewer's claims. The log line's text did change, which the brief allows.", "meanScore": 8, "durationMs": 15331, "costUsd": 0.034781, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 7751, "cache_read_input_tokens": 830, "output_tokens": 369, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 7751, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 369, "cache_read_input_tokens": 830, "cache_creation_input_tokens": 7751, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 7751 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 369, "cacheReadInputTokens": 830, "cacheCreationInputTokens": 7751, "webSearchRequests": 0, "costUSD": 0.034781, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 7751, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-codemod-sweep / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 31m 56s; budget used: 14.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 501237`; `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":2681,\"score\":32,\"scoreMax\":32,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.14173461125986322`; `facts.toolUse.totalToolCalls: 49`; `facts.timing.timeToFirstArtifactMs: 501237`; `facts.outcome.durationMs: 1915812` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15671 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31515 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.79 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 59835 + 14564 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 1 / 0 | `continuity.steps` |\n| Median step wall-clock | 194s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 13% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 501237`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":2681,\"score\":32,\"scoreMax\":32,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.14173461125986322`\n- `facts.toolUse.totalToolCalls: 49`\n- `facts.timing.timeToFirstArtifactMs: 501237`\n- `facts.outcome.durationMs: 1915812`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 501237`; `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":2681,\"score\":32,\"scoreMax\":32,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.14173461125986322`; `facts.toolUse.totalToolCalls: 49`; `facts.timing.timeToFirstArtifactMs: 501237`; `facts.outcome.durationMs: 1915812`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.5/10** remains unchanged. Advisory **8.33/10**, codex-inline/current-session.\n\nReal pre-edit enumeration covers7edits and4exclusions. All changed source files contain only the rename; protected fixtures are unchanged. Counts reconcile and both test runs pass3/3. Verification checks files manually but presents a fresh search without a receipt; its filtered tool surface omitted grep_files. One review-format rejection was repaired.\n\n| Advisory axis | Score | Evidence |\n|---|---:|---|\n| Enumeration completeness | 8 | A real pre-edit ripgrep search identifies all11sites, including re-export and log text; the7edits and4documented exclusions match the fixture. |\n| Invariant discipline | 9 | All4changed files equal the literal requested rename, with all4protected inputs byte-identical. No source behavior drift is observed. |\n| Reconciled verification | 8 | Counts reconcile and both real test runs pass3/3. Manual file inspection and the separate harness oracle support zero residuals; the claimed fresh search is not recorded because grep_files was filtered out during verification. |\n\nRead the complete2895-byte declared verification artifact and three supporting reports, all source/test fixtures and brief. Compared each non-harness fixture byte-for-byte or against the literal rename. Inspected every search, edit and test call, plus model-facing tool surfaces and gate repair. No extra trial or test was run.\n\n## Positives\n\n- All32 deterministic checks pass and all5craftbook stages complete. Four task-scoped workers hand off through the real workflow, with the apply worker reused for verify (outcome.success; sniff.latest=32/32; continuity.steps.completed=5; continuity.sessions.taskScoped=4).\n- Real pre-edit searches find all11sites. Seven replacements across4files handle the export, re-export, imports, calls and template-literal log. Two changelog and two brief occurrences are explicitly excluded (toolUse.byTool.grep_files=3; toolUse.byTool.replace_in_file=4; advisory-consistency-audit.json).\n- All4changed source/test files equal the seeded text with only fetchUserData→loadUser; package.json, src/log.js, CHANGELOG.md and the brief remain byte-identical. Both package-script runs pass the3tests, and the verification quotes the second receipt accurately (toolUse.byTool.run_package_script=2; sniff.latest; fixture audit).\n\n## Negatives\n\n- Verification says it reran the exact search, but every grep_files call precedes the edits. The worker instead lists and reads the tree. Its model-facing16-tool verification surface has no grep_files, although the prior apply surface does. The actual residual result is correct, but method provenance is overstated (toolUse.byTool.grep_files=3, read_files=3, list_dir=8; daemon.log02:44:08.777Z; verification.md).\n- The review initially includes a no-findings placeholder severity em dash; the gate treats it as an invalid finding and requires a rewrite. Removing that placeholder lets evaluate complete (continuity.steps.gateRejections=1; toolUse.byTool.advance_task_step=3; recorded gate rejection02:59:18Z).\n- Forty-nine tool calls and four workers are substantial overhead for7mechanical replacements. A second full test run is real and useful verification, but repeated full reports and a formatting-only review repair add cost (toolUse.totalToolCalls=49; continuity.sessions.total=4; outcome.durationMs=1915812).\n\n## Performance and GPU fit\n\nElapsed **31m56s**, 49 tool calls, **14564** engine-generated tokens at **8.79tok/s**, prefill 387.18tok/s (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10646/12227MiB**, process-tree RSS **15671MiB**, whole-system RAM **31515/31865MiB** (perf.gpu; perf.process; perf.systemMemory). System RAM includes other applications; shared GPU memory and paging are unmeasured.\n\nOne native0.1.57CUDA launch,65536context,one slot,f16KV. DenseFFN39/64GPU and25CPU: hybrid residency, not full GPU. Maximum captured session fill 13.4%; largest observed released native slot 13780tokens (21.0%). No observed compaction, endpoint failure, explicit OOM or context overflow. Context observations are not continuous allocation peaks. No new campaign memory peak.\n\n## Tactical experiment — not applied\n\nExperiment in ../gilde/data/craftbook-templates/co/codemod-sweep/versions/1.0.6/craftbook.json, verify.prompt: append “If the exact search tool is unavailable, state that limitation and label file inspection as manual verification; do not present it as a search receipt.” In evaluate.prompt require “For zero findings, write No findings and omit the findings table.” These are provenance/format instructions, not a solution to the seeded rename (toolUse.byTool.grep_files; continuity.steps.gateRejections). No catalog content changed.\n\n## Strategic experiments — not applied\n\nIn packages/core/src/prompt/role-tool-filter.ts and packages/core/src/prompt/prompt-tool-contract.ts, ensure workspace search tools explicitly required by the active step survive filtering; verify this through the emitted model-facing surface. Bind verification claims to tool receipt IDs/content hashes in packages/service/src/chat/manager.ts. In packages/script-stdlib/scripts/checkFixReview.ts, consider a distinct zero-findings representation rather than interpreting an empty placeholder as a finding. Measure false rejections separately from semantic misses.\n\n## Generalization and next experiment\n\nR1 and R2 both fixed8.5/advisory8.33. Both produce correct mechanical code and reconciled counts while relying on manual post-edit inspection rather than a recorded residual grep. R2 exposes a concrete missing verification search tool. Unrelated A/B reporting and API/security reviews also show that report assertions can exceed recorded validation; large-PR R2 shows stronger receipt-backed coverage. Same frozen native/tuning identity; no change-effect attribution.\n\nTest explicit tool availability and receipt-backed verification across corpus review, codemod and an unrelated report task. Measure unsupported validation claims, missing required tools, repair latency and task outcomes; leave all current gates/tuning untouched. Scenario-specific line-range acceptance would be an evaluator correction for future identity, not model prompt training.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/facts.json): pass, fixed8.5, 28.09min, 8.8tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/facts.json): fail, fixed5.3, 16.80min, 119.22tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-50-57-501Z-evbb/facts.json): fail, fixed4.6, 7.99min, 109.44tok/s.\n\nOther-model runs are contextual and not a controlled native-version comparison.\n\n## Strategic note\n\nSuccessful loading and fixed gates do not establish all claimed work was executed. Preserve the coverage/provenance distinctions above. No campaign rerun, source edit, model download, tuning change or gate change was made.\n\n## Raw evidence\n\nfacts.json,score.json,advisory-artifact-review.json,advisory-consistency-audit.json,advisory-tool-evidence.json,historical-comparison.json and the captured artifacts/workspace.\n" }, { "cellId": "r2-saluki-cuda-developer-craftbook-refactor-module", "baselineCellId": "r2-saluki-cuda-developer-craftbook-refactor-module", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T03:07:31.672Z", "finishedAt": "2026-11-06T04:09:28.739Z", "durationMs": 3717067, "success": true, "reason": "craftbook-refactor-module passed 30 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 382411 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 30, "scoreMax": 30, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.20624559440273862 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 66 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 382411 }, { "source": "facts", "path": "outcome.durationMs", "value": 3717067 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 624, "peakRssMb": 15004, "peakCpuPercent": 1333.7, "firstAtMs": 0, "lastAtMs": 3703431 }, "gpu": { "available": true, "sampleCount": 624, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 624, "peakUsedMb": 31585, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 97627, "totalOutputTokens": 27884, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 115931, "todayTokensCached": 111051, "todayTokensOut": 8100, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 115931, "totalTokensCached": 111051, "totalTokensOut": 8100, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13380, "cachedInputTokens": 12808, "outputTokens": 79, "cost": 0, "durationMs": 161132, "outputTokensPerSec": 8.512236722247572, "at": "2026-11-06T04:09:20.692Z" }, "lastUpdated": "2026-11-06T04:09:20.692Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T04:09:23.359Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3705269, "meanTokensPerSec": 8.39, "genTokensPerSec": 8.39, "genTokens": 27884, "promptTokensPerSec": 367.95, "peakRssMb": 15004, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 115931, "output": 8100 } } } }, "craftbook": { "craftbookId": "refactor-module", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 8, "completed": 6, "gateApprovals": 6, "gateRejections": 3, "redrives": 0, "perStep": [ { "ref": "crewlog/1", "stepId": "baseline", "activatedAt": "2026-11-06T03:17:03.791Z", "completedAt": "2026-11-06T03:20:02.963Z", "ms": 179172, "gateRejections": 1, "toolCalls": 8, "sessionIds": [ "34d6576f-c914-4fd8-b4ca-6a3b98ef0f78" ] }, { "ref": "crewlog/1", "stepId": "plan", "activatedAt": "2026-11-06T03:20:02.966Z", "completedAt": "2026-11-06T03:23:44.301Z", "ms": 221335, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "b5d86dc9-6e82-497d-a469-430af1e9a5fb" ] }, { "ref": "crewlog/1", "stepId": "change", "activatedAt": "2026-11-06T03:29:03.926Z", "completedAt": "2026-11-06T03:32:45.568Z", "ms": 221642, "gateRejections": 1, "toolCalls": 22, "sessionIds": [ "fc5798fe-f969-4285-8600-46bab5b6da47" ] }, { "ref": "crewlog/1", "stepId": "verify", "activatedAt": "2026-11-06T03:32:45.569Z", "completedAt": "2026-11-06T03:39:26.692Z", "ms": 401123, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "fc5798fe-f969-4285-8600-46bab5b6da47" ] }, { "ref": "crewlog/1", "stepId": "evaluate", "activatedAt": "2026-11-06T03:58:50.950Z", "completedAt": "2026-11-06T04:06:34.190Z", "ms": 463240, "gateRejections": 1, "toolCalls": 21, "sessionIds": [ "22d89633-7b57-4e79-8c5f-3a6f6bafee6a" ] }, { "ref": "crewlog/1", "stepId": "finish", "activatedAt": "2026-11-06T04:06:34.192Z", "completedAt": "2026-11-06T04:09:20.635Z", "ms": 166443, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "72557461-c471-411b-9408-1ce0ffab56f2" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T03:07:37.870Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T03:07:37.761Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 221488.5 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "ebou": 2, "elara": 1, "moussa": 2, "valentino": 1, "vincenzo": 1 }, "perTask": { "crewlog/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 5, "maxContextFill": 0.1818695068359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1818695068359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-refactor-module passed 30 deterministic craftbook checks", "durationMs": 3717067, "timeoutMs": 18022528, "budgetUsedFraction": 0.20624559440273862 }, "timing": { "startedAt": "2026-11-06T03:07:31.672Z", "finishedAt": "2026-11-06T04:09:28.739Z", "timeToFirstArtifactMs": 382411, "timeToLastArtifactWriteMs": 3542550, "timeToFirstTokenMs": 37636, "firstTurnTtftMs": 23312, "timeToFirstToolCallMs": 50423 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Developer", "Software Architect", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 66, "byTool": { "read_files": 3, "run_package_script": 7, "write_file": 4, "write_artifact": 10, "replace_in_file": 3, "read_artifact": 16, "read_file": 10, "replace_lines": 1, "list_dir": 3, "stat": 1, "validate": 3, "grep_artifact": 1, "advance_task_step": 3, "write_task_note": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 18 }, "sniff": { "progression": [ { "atMs": 8358, "filePath": "craftbook-refactor-module", "bytes": 0, "score": 5, "scoreMax": 30, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/format-name.js, src/report.js, tests/format.test.mjs, tests/report.test.mjs, source/refactor-brief.md" }, { "atMs": 54787, "filePath": "craftbook-refactor-module", "bytes": 0, "score": 6, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 572797, "filePath": "craftbook-refactor-module", "bytes": 3574, "score": 10, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 757356, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 11, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 977444, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 15, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1072201, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 15, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1137413, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 16, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1185303, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 18, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1292363, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 21, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1298357, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 21, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1517933, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 22, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 1915989, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 26, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 3063385, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 28, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 3081477, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 28, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 3699553, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 29, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" }, { "atMs": 3711169, "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 30, "scoreMax": 30, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-refactor-module", "bytes": 3592, "score": 30, "scoreMax": 30, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T03:07:31.676Z] [trial] id=craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c scenario=craftbook-refactor-module model=underdog-saluki-27b-iq2-mix", "[2026-11-06T03:07:31.676Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "[2026-11-06T03:07:31.676Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T03:07:31.676Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-refactor-module-QHlyQr", "[2026-11-06T03:07:31.676Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T03:07:31.679Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T03:07:31.683Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T03:07:31.690Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-refactor-module-QHlyQr", "[2026-11-06T03:07:31.690Z] [trial] throughput-scaled ceiling: 120m → 300m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T03:07:37.857Z] [trial] daemon spawned pid=52220 port=https://127.0.0.1:60106", "[2026-11-06T03:07:38.991Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-06T03:07:39.008Z] [trial] running scenario setup hook", "[2026-11-06T03:07:39.008Z] [trial] meester=elara", "[2026-11-06T03:07:39.625Z] [trial] scenario setup complete", "[2026-11-06T03:07:39.627Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T03:07:39.762Z] [poll] initial digests hard=1047a0c1112903c4 soft=6f6bd7440ce45a09", "[2026-11-06T03:07:40.077Z] [poll] hard-progress: 1047a0c1112903c4 → 439bed4fa0fb0acb after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-06T03:07:45.364Z] [poll] hard-progress: 439bed4fa0fb0acb → 8eb1774435c989a5 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-06T03:08:26.532Z] [poll] hard-progress: 8eb1774435c989a5 → 6107a96f3f2477ae after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=46,stream=162) sniff=6/0B)", "[2026-11-06T03:08:44.205Z] [poll] hard-progress: 6107a96f3f2477ae → 9cc782389069f1b2 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=82,stream=264) sniff=6/0B)", "[2026-11-06T03:13:54.321Z] [poll] hard-progress: 9cc782389069f1b2 → 9dd3eb836e85973d after 310s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=918,stream=3310) sniff=6/0B)", "[2026-11-06T03:14:05.892Z] [poll] hard-progress: 9dd3eb836e85973d → 012d0e803c9a92b7 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=929,stream=3340) sniff=6/0B)", "[2026-11-06T03:17:04.528Z] [poll] hard-progress: 012d0e803c9a92b7 → f98309ca647dfdf3 after 179s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=1406,stream=5673) sniff=10/3574B)", "[2026-11-06T03:17:51.979Z] [poll] hard-progress: f98309ca647dfdf3 → e16bf0a0391b05ca after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=1508,stream=6067) sniff=10/3574B)", "[2026-11-06T03:20:03.014Z] [poll] hard-progress: e16bf0a0391b05ca → a2efb7515f286172 after 131s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=1859,stream=8040) sniff=10/3574B)" ] }, "metrics": { "process": { "sampleCount": 624, "peakRssMb": 15004, "peakCpuPercent": 1333.7, "firstAtMs": 0, "lastAtMs": 3703431 }, "gpu": { "available": true, "sampleCount": 624, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 624, "peakUsedMb": 31585, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 97627, "totalOutputTokens": 27884, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 115931, "todayTokensCached": 111051, "todayTokensOut": 8100, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 115931, "totalTokensCached": 111051, "totalTokensOut": 8100, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13380, "cachedInputTokens": 12808, "outputTokens": 79, "cost": 0, "durationMs": 161132, "outputTokensPerSec": 8.512236722247572, "at": "2026-11-06T04:09:20.692Z" }, "lastUpdated": "2026-11-06T04:09:20.692Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T04:09:23.359Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3705269, "meanTokensPerSec": 8.39, "genTokensPerSec": 8.39, "genTokens": 27884, "promptTokensPerSec": 367.95, "peakRssMb": 15004, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 115931, "output": 8100 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T03:07:31.672Z", "finishedAt": "2026-11-06T04:09:28.739Z", "timeToFirstArtifactMs": 382411, "timeToLastArtifactWriteMs": 3542550, "timeToFirstTokenMs": 37636, "firstTurnTtftMs": 23312, "timeToFirstToolCallMs": 50423 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 8, "completed": 6, "gateApprovals": 6, "gateRejections": 3, "redrives": 0, "perStep": [ { "ref": "crewlog/1", "stepId": "baseline", "activatedAt": "2026-11-06T03:17:03.791Z", "completedAt": "2026-11-06T03:20:02.963Z", "ms": 179172, "gateRejections": 1, "toolCalls": 8, "sessionIds": [ "34d6576f-c914-4fd8-b4ca-6a3b98ef0f78" ] }, { "ref": "crewlog/1", "stepId": "plan", "activatedAt": "2026-11-06T03:20:02.966Z", "completedAt": "2026-11-06T03:23:44.301Z", "ms": 221335, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "b5d86dc9-6e82-497d-a469-430af1e9a5fb" ] }, { "ref": "crewlog/1", "stepId": "change", "activatedAt": "2026-11-06T03:29:03.926Z", "completedAt": "2026-11-06T03:32:45.568Z", "ms": 221642, "gateRejections": 1, "toolCalls": 22, "sessionIds": [ "fc5798fe-f969-4285-8600-46bab5b6da47" ] }, { "ref": "crewlog/1", "stepId": "verify", "activatedAt": "2026-11-06T03:32:45.569Z", "completedAt": "2026-11-06T03:39:26.692Z", "ms": 401123, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "fc5798fe-f969-4285-8600-46bab5b6da47" ] }, { "ref": "crewlog/1", "stepId": "evaluate", "activatedAt": "2026-11-06T03:58:50.950Z", "completedAt": "2026-11-06T04:06:34.190Z", "ms": 463240, "gateRejections": 1, "toolCalls": 21, "sessionIds": [ "22d89633-7b57-4e79-8c5f-3a6f6bafee6a" ] }, { "ref": "crewlog/1", "stepId": "finish", "activatedAt": "2026-11-06T04:06:34.192Z", "completedAt": "2026-11-06T04:09:20.635Z", "ms": 166443, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "72557461-c471-411b-9408-1ce0ffab56f2" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T03:07:37.870Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T03:07:37.761Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 221488.5 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "ebou": 2, "elara": 1, "moussa": 2, "valentino": 1, "vincenzo": 1 }, "perTask": { "crewlog/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 5, "maxContextFill": 0.1818695068359375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1818695068359375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-craftbook-refactor-module", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "scenarioId": "craftbook-refactor-module", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-06T04:11:44.298Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/artifacts/crewlog/tasks/1/change-notes.md", "artifactBytes": 1744, "artifactSha256": "df69bb8690f5c9d01a1b3ee0981558d51b338e5a3bdfae352cbf088ed2040105", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/re/refactor-module/versions/2.0.7/test.json", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ], "scoreAxes": { "Guardrail-first discipline": 8, "Reversible-stage execution": 8, "Genuine deduplication": 9 }, "axisJustifications": { "Guardrail-first discipline": "A real2-test baseline and13new characterization tests run green before restructuring. Duplication is correctly identified, but supporting function-length measurements are inconsistent.", "Reversible-stage execution": "Three planned source changes each have a separate15/15passing receipt. The notes falsely list the untouched original test files as changed; no stage result is invented.", "Genuine deduplication": "The shared implementation exactly preserves the original body. Both original callers delegate to it, public exports survive, original tests remain intact and stage/verification runs pass." }, "meanScore": 8.33, "justification": "Real green baselines and13new characterization tests precede restructuring. Each of3planned stages has its own15/15passing receipt. The shared implementation exactly preserves the old body and public exports. Notes incorrectly list untouched original tests as changed, and supporting function-length measurements mix conventions.", "reviewBasis": "Read all5complete documents, all3finalsourcefiles, all4tests and the seeded brief. Matched source/test changes against the frozen fixture and inspected the full test/edit chronology. No new test or model run. The standard runner completed all6craftbook stages and30fixed checks.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/baseline.md", "originalPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/artifacts/crewlog/tasks/1/baseline.md", "bytes": 3628, "sha256": "59b4b17b08baf8ed76ef668eb72d92fe4408ff25eaf07ea5bff077970417d60b", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/plan.md", "originalPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/artifacts/crewlog/tasks/1/plan.md", "bytes": 4199, "sha256": "047bfbff81a8664c5e0835f990c8ea36a57b1fc3341f7bf0fe4bfdaa4099f0ee", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/change-notes.md", "originalPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/artifacts/crewlog/tasks/1/change-notes.md", "bytes": 1744, "sha256": "df69bb8690f5c9d01a1b3ee0981558d51b338e5a3bdfae352cbf088ed2040105", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/verification.md", "originalPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/artifacts/crewlog/tasks/1/verification.md", "bytes": 4571, "sha256": "6eb12dc8dac471f46554d4a6bcd38b9bb05ebfa8bf5a4d0d0b4cea1c264abd67", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/review.md", "originalPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/artifacts/crewlog/tasks/1/review.md", "bytes": 6208, "sha256": "b33dec6fa53b1260206bc70f98832ca188ec93bdbb08ffd03040b57fca28b0a5", "status": "reviewedSupportingArtifact" } ], "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 26032, "nativeFailureLines": [], "latestGeneration": "2026-11-06T04:09:05.900Z INFO [chat] [llama-server] 61.18.603.060 I slot print_timing: id 0 | task 28055 | n_gen = 592, tg = 8.41 t/s, tg_3s = 8.41 t/s", "gpuPointMiB": 10646, "gpuTotalMiB": 12227, "wholeSystemRamPointMiB": 27283, "wholeSystemRamTotalMiB": 31865, "pointObservedAt": "2026-11-06T04:07:08.110Z", "note": "Point samples are not peaks. CPU offload is explicit; sharedGPUmemory/paging unmeasured. No observed native endpoint/OOM/context failure.", "sampledPeaks": { "gpuMiB": 10648, "processTreeRssMiB": 15004, "wholeSystemRamMiB": 31585 } } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:06.860Z", "reviewId": "076", "cellId": "r2-saluki-cuda-developer-craftbook-refactor-module", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "scenarioId": "craftbook-refactor-module", "status": "scored", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "baselineCellId": "r2-saluki-cuda-developer-craftbook-refactor-module", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "d97c75ee463e5a2776bf7e3f9a60e253761b7382ad06767476501f995f9b55d7", "evidenceSha256": "13294fbd85ab6302fe5ad3731a5ad80be63908ff4146c1fec24b5489fcfcf300", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/artifacts/crewlog/tasks/1/change-notes.md", "bytes": 1744, "sha256": "df69bb8690f5c9d01a1b3ee0981558d51b338e5a3bdfae352cbf088ed2040105", "shownBytes": 1744, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/baseline.md", "bytes": 3628, "sha256": "59b4b17b08baf8ed76ef668eb72d92fe4408ff25eaf07ea5bff077970417d60b", "shownBytes": 3628, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/plan.md", "bytes": 4199, "sha256": "047bfbff81a8664c5e0835f990c8ea36a57b1fc3341f7bf0fe4bfdaa4099f0ee", "shownBytes": 4199, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/change-notes.md", "bytes": 1744, "sha256": "df69bb8690f5c9d01a1b3ee0981558d51b338e5a3bdfae352cbf088ed2040105", "shownBytes": 1744, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/verification.md", "bytes": 4571, "sha256": "6eb12dc8dac471f46554d4a6bcd38b9bb05ebfa8bf5a4d0d0b4cea1c264abd67", "shownBytes": 4571, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/review.md", "bytes": 6208, "sha256": "b33dec6fa53b1260206bc70f98832ca188ec93bdbb08ffd03040b57fca28b0a5", "shownBytes": 6208, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c\\workspace\\crewlog\\source\\refactor-brief.md", "bytes": 720, "sha256": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "shownBytes": 720, "truncated": false } ], "scoreAxes": { "Guardrail-first discipline": 8, "Reversible-stage execution": 7, "Genuine deduplication": 7 }, "justification": "Baseline records 2/2 green, then characterization tests before any source edit, with a stated exact-body-comparison method and a guardrail. Three planned stages each have a 15/15 receipt and no deviations. Dedup is claimed and reviewed, but the source files are not shown. Change-notes list unmodified tests as \"touched\".", "limitations": "The actual src/ and tests/ files and raw per-stage run logs were not in the evidence. Dedup and API preservation rest on the agent's and reviewer's descriptions. The baseline never says whether any bugs were pinned. No runtime was re-executed.", "meanScore": 7.333333333333333, "durationMs": 18170, "costUsd": 0.057564, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 12419, "cache_read_input_tokens": 840, "output_tokens": 780, "output_tokens_details": { "thinking_tokens": 428 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 12419, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 780, "cache_read_input_tokens": 840, "cache_creation_input_tokens": 12419, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 12419 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 780, "cacheReadInputTokens": 840, "cacheCreationInputTokens": 12419, "webSearchRequests": 0, "costUSD": 0.057564, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 428, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 12419, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-refactor-module / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 61m 57s; budget used: 20.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 382411`; `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":3592,\"score\":30,\"scoreMax\":30,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.20624559440273862`; `facts.toolUse.totalToolCalls: 66`; `facts.timing.timeToFirstArtifactMs: 382411`; `facts.outcome.durationMs: 3717067` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15004 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10648 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31585 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.39 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 97627 + 27884 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 8 → 6 | `continuity.steps` |\n| Gate rejections / redrives | 3 / 0 | `continuity.steps` |\n| Median step wall-clock | 221s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.63 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 18% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 382411`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":3592,\"score\":30,\"scoreMax\":30,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.20624559440273862`\n- `facts.toolUse.totalToolCalls: 66`\n- `facts.timing.timeToFirstArtifactMs: 382411`\n- `facts.outcome.durationMs: 3717067`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 382411`; `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":3592,\"score\":30,\"scoreMax\":30,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.20624559440273862`; `facts.toolUse.totalToolCalls: 66`; `facts.timing.timeToFirstArtifactMs: 382411`; `facts.outcome.durationMs: 3717067`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS8.5/10**, all30checks, remains unchanged. Advisory **8.33/10**, codex-inline/current-session.\n\n| Axis | Score | Evidence |\n|---|---:|---|\n| Guardrail-first discipline | 8 | A real2-test baseline and13new characterization tests run green before restructuring. Duplication is correctly identified, but supporting function-length measurements are inconsistent. |\n| Reversible-stage execution | 8 | Three planned source changes each have a separate15/15passing receipt. The notes falsely list the untouched original test files as changed; no stage result is invented. |\n| Genuine deduplication | 9 | The shared implementation exactly preserves the original body. Both original callers delegate to it, public exports survive, original tests remain intact and stage/verification runs pass. |\n\nReal green baselines and13new characterization tests precede restructuring. Each of3planned stages has its own15/15passing receipt. The shared implementation exactly preserves the old body and public exports. Notes incorrectly list untouched original tests as changed, and supporting function-length measurements mix conventions.\n\nRead all5complete documents, all3finalsourcefiles, all4tests and the seeded brief. Matched source/test changes against the frozen fixture and inspected the full test/edit chronology. No new test or model run. The standard runner completed all6craftbook stages and30fixed checks.\n\n## Positives\n\n- All30 deterministic checks pass, with all6craftbook stages complete and5task-scoped sessions. This is real workflow execution, including the final DONE note and finish transition (outcome.success; sniff.latest; continuity.steps.completed=6; continuity.sessions.taskScoped=5).\n- A real2/2baseline precedes13new characterization tests, followed by a15/15green baseline before any source restructure. Tests exercise both public call paths, ALL-CAPS, repeated spaces, empty strings, whitespace-only input and summary cases (toolUse.byTool.run_package_script=7; toolUse.byTool.write_file=4; advisory-consistency-audit.json).\n- Each of3planned edits has a separate15/15passing run before the next edit. The final verification run also passes15/15. Unlike R1, no separate stage1receipt is invented (toolUse.byTool.run_package_script=7; stageEvidence in advisory-consistency-audit.json).\n- The new shared module equals the original implementation with only its function name changed. Both original callers delegate, the public exports survive, original test files/package/brief are byte-identical, and one title-case body remains (outcome.reason; artifacts.otherFileCount; fixture comparisons and captured source).\n\n## Negatives\n\n- The change notes list tests/format.test.mjs and tests/report.test.mjs as touched even though both are byte-identical to the fixture and no writes target them. The reviewer checks existence and accepts the list rather than checking the actual diff (artifacts.otherFileCount; toolUse.byTool.write_file and replace_in_file; unchangedFilesIncorrectlyListedAsTouched in audit).\n- Measurement prose counts an8-line function as7lines; verification compares3whole-function lines before with1bodyline after for reportLine/reportSummary, whose function lengths actually remain3. The genuine2-to-1duplicate-body reduction is separate from those inconsistent metrics (artifacts.otherFileCount; captured baseline/verification and source; audit.measurementNotes).\n- Three craftbook gate rejections add repair overhead: a proposed future file is rejected as nonexistent in the baseline, a valid source-relative import ./normalize-name.js is treated as a workspace-root citation in change notes, and a zero-findings severity placeholder is rejected in review (continuity.steps.gateRejections=3; recording/task-notes.json).\n- The legitimate119-byte replacement for the240-byte format-name.js is refused by the source shortening guard, then accepted as an explicit targeted replacement. A workspace-only edit of an artifact also fails. Two reviewer writes are rejected for unsupported or uncited details; later rewrites remove metadata and add citations (toolUse.byTool.write_file=4, replace_in_file=3, write_artifact=10; advisory-tool-evidence.json).\n- Sixty-six tool calls and61m57s are substantial cost for extracting a six-line body, despite useful characterization coverage. The lengthy review and repeated document repairs dominate work after the source change is already correct (toolUse.totalToolCalls=66; outcome.durationMs=3717067; continuity.steps.perStep; recorded write/test chronology).\n\n## Performance and GPU fit\n\nElapsed **61m57s**,66calls,27,884engine-generated tokens at **8.39tok/s**, prefill367.95tok/s (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10,648/12,227MiB**, process-tree RSS **15,004MiB**, whole-system RAM **31,585/31,865MiB** (perf.gpu; perf.process; perf.systemMemory). Whole-system memory includes other applications. Shared GPU memory and paging were not measured. No new campaign peak.\n\nOne native0.1.57CUDA launch,65,536context,one slot,f16KV, denseFFN39/64layers onGPU and25CPU: hybrid residency. Captured session fill peaks18.2%; largest observed released native slot26032tokens (39.7%). No compaction, endpoint failure, explicit OOM or context overflow observed. Native slot observations are not continuous KV peaks.\n\n## Tactical experiments — not applied\n\nIn ../gilde/data/craftbook-templates/re/refactor-module/versions/2.0.7/craftbook.json, change.prompt, add “Derive Files touched from successful source writes or an actual diff; list inspected-but-unchanged files separately. Use targeted replacements for extraction edits.” In baseline/verify prompts require one consistent measurement definition, e.g. “body lines exclude signature and closing brace; total function lines include both.” These directly address the touched-file overclaim and mixed measurement conventions (artifacts.otherFileCount; toolUse.byTool.write_file).\n\nFor evaluate.prompt, use “When there are no findings, write No findings and omit an empty findings row.” Measure avoidable review repairs across refactor and codemod, not just whether this fixture passes (continuity.steps.gateRejections=3). No sampling, prompts or catalog data changed.\n\n## Strategic experiments — not applied\n\nThe baseline gate in the same craftbook should distinguish future planned output paths from evidence about existing files. The change gate should resolve a quoted import relative to its cited source module rather than automatically treating it as a workspace-root file. Make these semantic distinctions in a future separately versioned gate; preserve this campaign’s identity.\n\nInspect packages/core/src/local-loop/wrappers/source-write-guard.ts for a targeted-edit recommendation on large valid extraction diffs; do not globally relax truncation protection. In packages/core/src/checks/claims.ts, evaluate quotation/measurement provenance independently of prose formatting. In packages/script-stdlib/scripts/checkFixReview.ts, give zero-findings reports a distinct accepted representation. Bind touched-file claims to edit receipts/content hashes in packages/service/src/chat/manager.ts, rather than verifying only that named files exist.\n\n## Generalization and next experiment\n\nR1 fixed8.5/advisory7.67 (8/6/9) had real baseline coverage and a correct final extraction but invented a separate stage1green receipt. R2 fixed8.5/advisory8.33 (8/8/9) has a genuine test run after each stage and13new characterization tests. The same false touched-file claim persists. This is repetition variability under unchanged native/tuning identity, not an improvement attributable to a code change.\n\nUnrelated A/B reporting and API review likewise separate plausible prose from actual evidence. Codemod repeats the zero-findings severity rejection and has a concrete filtered-search-tool gap. Test receipt-bound reporting, planned-path semantics and zero-findings handling across these tasks; measure unsupported claims, false rejections, tool cost and preserved task outcomes. Do not encode fixture names or solutions in model-specific prompts.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/facts.json): pass, fixed8.5, 59.79min, 8.6tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/facts.json): fail, fixed5, 26.51min, 120.37tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-27-37-405Z-h3wg/facts.json): fail, fixed4.7, 13.40min, 129.41tok/s.\n\nOther-model history is contextual, not a controlled native-version comparison.\n\n## Strategic note\n\nThe refactor and stage tests are genuine; the remaining issues are evidence reporting and guard-induced overhead. The final30-check result includes a separate harness behavior/deduplication oracle. No additional test execution, model retry, code/tuning/gate edit or download was performed by the monitor.\n\n## Raw evidence\n\nfacts.json,score.json,advisory-artifact-review.json,advisory-consistency-audit.json,advisory-tool-evidence.json,historical-comparison.json,recording/task-notes.json and captured artifacts/workspace.\n" }, { "cellId": "r2-saluki-cuda-developer-craftbook-bug-fix-tdd", "baselineCellId": "r2-saluki-cuda-developer-craftbook-bug-fix-tdd", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T04:12:52.340Z", "finishedAt": "2026-11-06T04:38:00.602Z", "durationMs": 1508262, "success": true, "reason": "craftbook-bug-fix-tdd passed 28 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 105947 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06695007492844511 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 38 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 105947 }, { "source": "facts", "path": "outcome.durationMs", "value": 1508262 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 252, "peakRssMb": 14900, "peakCpuPercent": 1207.3, "firstAtMs": 1, "lastAtMs": 1493891 }, "gpu": { "available": true, "sampleCount": 252, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 252, "peakUsedMb": 31043, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 63566, "totalOutputTokens": 10591, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 63482, "todayTokensCached": 59649, "todayTokensOut": 2930, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 63482, "totalTokensCached": 59649, "totalTokensOut": 2930, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8469, "cachedInputTokens": 7894, "outputTokens": 97, "cost": 0, "durationMs": 108881, "outputTokensPerSec": 9.168022929225346, "at": "2026-11-06T04:37:51.314Z" }, "lastUpdated": "2026-11-06T04:37:51.314Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T04:37:55.106Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1496531, "meanTokensPerSec": 8.91, "genTokensPerSec": 8.91, "genTokens": 10591, "promptTokensPerSec": 402.92, "peakRssMb": 14900, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 63482, "output": 2930 } } } }, "craftbook": { "craftbookId": "bug-fix-tdd", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 6, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "orderly-pricing/1", "stepId": "reproduce", "activatedAt": "2026-11-06T04:12:59.872Z", "completedAt": "2026-11-06T04:16:00.576Z", "ms": 180704, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "b9a04dfc-33bf-4d97-abce-39b47f702039" ] }, { "ref": "orderly-pricing/1", "stepId": "locate", "activatedAt": "2026-11-06T04:16:00.577Z", "completedAt": "2026-11-06T04:21:22.571Z", "ms": 321994, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "b1f6f866-2d82-4a8e-be30-a0cbd8bec592" ] }, { "ref": "orderly-pricing/1", "stepId": "fix", "activatedAt": "2026-11-06T04:21:22.572Z", "completedAt": "2026-11-06T04:26:22.717Z", "ms": 300145, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "e2b228fc-5960-4805-b75f-72eb90bd79ea" ] }, { "ref": "orderly-pricing/1", "stepId": "validate", "activatedAt": "2026-11-06T04:26:22.719Z", "completedAt": "2026-11-06T04:28:27.017Z", "ms": 124298, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "e2b228fc-5960-4805-b75f-72eb90bd79ea" ] }, { "ref": "orderly-pricing/1", "stepId": "evaluate", "activatedAt": "2026-11-06T04:28:27.018Z", "completedAt": "2026-11-06T04:34:56.888Z", "ms": 389870, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "c665e12f-6506-4bdc-b6b3-99cf703f157f" ] }, { "ref": "orderly-pricing/1", "stepId": "finish", "activatedAt": "2026-11-06T04:34:56.889Z", "completedAt": "2026-11-06T04:37:51.273Z", "ms": 174384, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "0ab053b7-768a-4094-a8fc-86d4c7398ba9" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T04:12:58.249Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T04:12:58.143Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 240424.5 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "alvaro": 1, "aysel": 2, "mahnoor": 2, "pavlina": 1, "tamsin": 1 }, "perTask": { "orderly-pricing/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.14501953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.14501953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-bug-fix-tdd passed 28 deterministic craftbook checks", "durationMs": 1508262, "timeoutMs": 22528160, "budgetUsedFraction": 0.06695007492844511 }, "timing": { "startedAt": "2026-11-06T04:12:52.340Z", "finishedAt": "2026-11-06T04:38:00.602Z", "timeToFirstArtifactMs": 105947, "timeToLastArtifactWriteMs": 1324573, "timeToFirstTokenMs": 38255, "firstTurnTtftMs": 23042, "timeToFirstToolCallMs": 52844 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Developer", "Root-Cause Analyst", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 38, "byTool": { "read_file": 10, "append_to_file": 1, "run_package_script": 2, "write_artifact": 6, "read_artifact": 8, "list_dir": 2, "grep_files": 1, "replace_in_file": 1, "read_task_notes": 1, "read_files": 1, "validate": 2, "advance_task_step": 2, "write_task_note": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 13 }, "sniff": { "progression": [ { "atMs": 8140, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/pricing.js, tests/pricing.test.mjs, source/bug-report.md" }, { "atMs": 54303, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 6, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 107703, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 113367, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 119310, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 125191, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 131118, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 137018, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 142898, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 148750, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 154679, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 160576, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 166456, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 172387, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 178274, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 184165, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 189810, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 195777, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 201859, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 207993, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 213786, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 219742, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 225557, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 231533, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 237566, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 243370, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 249368, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 255354, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 261374, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 267308, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 273242, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 279221, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 285182, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 291235, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 297140, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 303094, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 309187, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 315142, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 321159, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 327212, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 333222, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 339289, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 345276, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 351280, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 357344, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 363412, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 369446, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 375476, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 381529, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 387517, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 393564, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 399564, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 405624, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 411592, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 417596, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 423626, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 429673, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 435719, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 441730, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 447770, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 453800, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 459885, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 465954, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 471972, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 478012, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 484080, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 490212, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 496235, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 502239, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 508320, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 514020, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 520285, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 526246, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 532270, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 538045, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 544019, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 549896, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 555921, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 561663, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 567580, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 573456, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 579385, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 585263, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 591145, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 597114, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 603382, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 16, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 812420, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 20, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 937266, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 23, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1315287, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 25, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1490674, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 27, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1502353, "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-bug-fix-tdd", "bytes": 1588, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T04:12:52.343Z] [trial] id=craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4 scenario=craftbook-bug-fix-tdd model=underdog-saluki-27b-iq2-mix", "[2026-11-06T04:12:52.343Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T04:12:52.343Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-bug-fix-tdd-qe6igr", "[2026-11-06T04:12:52.343Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "[2026-11-06T04:12:52.343Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T04:12:52.346Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T04:12:52.349Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T04:12:52.354Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-bug-fix-tdd-qe6igr", "[2026-11-06T04:12:52.355Z] [trial] throughput-scaled ceiling: 150m → 375m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T04:12:58.207Z] [trial] daemon spawned pid=39544 port=https://127.0.0.1:52831", "[2026-11-06T04:12:59.490Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-06T04:12:59.508Z] [trial] running scenario setup hook", "[2026-11-06T04:12:59.508Z] [trial] meester=tamsin", "[2026-11-06T04:13:00.076Z] [trial] scenario setup complete", "[2026-11-06T04:13:00.078Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T04:13:00.131Z] [poll] initial digests hard=1208feadfdf67f3f soft=a33c01d4a7c25094", "[2026-11-06T04:13:00.520Z] [poll] hard-progress: 1208feadfdf67f3f → 5ab0e2e85225d392 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-06T04:13:06.007Z] [poll] hard-progress: 5ab0e2e85225d392 → e4516b4d62d2ef9a after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-06T04:13:46.697Z] [poll] hard-progress: e4516b4d62d2ef9a → 4a21a2f6f124c58a after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=37,stream=81) sniff=6/0B)", "[2026-11-06T04:14:40.113Z] [poll] hard-progress: 4a21a2f6f124c58a → 2f62ab4ff8f792b4 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=177,stream=531) sniff=5/0B)", "[2026-11-06T04:14:45.762Z] [poll] hard-progress: 2f62ab4ff8f792b4 → 6776d85745caa656 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=183,stream=546) sniff=5/0B)", "[2026-11-06T04:16:02.203Z] [poll] hard-progress: 6776d85745caa656 → 5e30b46f98d3486e after 76s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=382,stream=1624) sniff=9/1588B)", "[2026-11-06T04:16:08.198Z] [poll] hard-progress: 5e30b46f98d3486e → 3da28ec2f8e29ef7 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=382,stream=1624) sniff=9/1588B)", "[2026-11-06T04:16:32.147Z] [poll] hard-progress: 3da28ec2f8e29ef7 → 561ffdf2a75ddbe6 after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=405,stream=1668) sniff=9/1588B)", "[2026-11-06T04:16:43.953Z] [poll] hard-progress: 561ffdf2a75ddbe6 → e7293a955105b7d8 after 12s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=10,slot=428,stream=1739) sniff=9/1588B)" ] }, "metrics": { "process": { "sampleCount": 252, "peakRssMb": 14900, "peakCpuPercent": 1207.3, "firstAtMs": 1, "lastAtMs": 1493891 }, "gpu": { "available": true, "sampleCount": 252, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 252, "peakUsedMb": 31043, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 63566, "totalOutputTokens": 10591, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 63482, "todayTokensCached": 59649, "todayTokensOut": 2930, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 63482, "totalTokensCached": 59649, "totalTokensOut": 2930, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8469, "cachedInputTokens": 7894, "outputTokens": 97, "cost": 0, "durationMs": 108881, "outputTokensPerSec": 9.168022929225346, "at": "2026-11-06T04:37:51.314Z" }, "lastUpdated": "2026-11-06T04:37:51.314Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T04:37:55.106Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1496531, "meanTokensPerSec": 8.91, "genTokensPerSec": 8.91, "genTokens": 10591, "promptTokensPerSec": 402.92, "peakRssMb": 14900, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 63482, "output": 2930 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T04:12:52.340Z", "finishedAt": "2026-11-06T04:38:00.602Z", "timeToFirstArtifactMs": 105947, "timeToLastArtifactWriteMs": 1324573, "timeToFirstTokenMs": 38255, "firstTurnTtftMs": 23042, "timeToFirstToolCallMs": 52844 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 6, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "orderly-pricing/1", "stepId": "reproduce", "activatedAt": "2026-11-06T04:12:59.872Z", "completedAt": "2026-11-06T04:16:00.576Z", "ms": 180704, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "b9a04dfc-33bf-4d97-abce-39b47f702039" ] }, { "ref": "orderly-pricing/1", "stepId": "locate", "activatedAt": "2026-11-06T04:16:00.577Z", "completedAt": "2026-11-06T04:21:22.571Z", "ms": 321994, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "b1f6f866-2d82-4a8e-be30-a0cbd8bec592" ] }, { "ref": "orderly-pricing/1", "stepId": "fix", "activatedAt": "2026-11-06T04:21:22.572Z", "completedAt": "2026-11-06T04:26:22.717Z", "ms": 300145, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "e2b228fc-5960-4805-b75f-72eb90bd79ea" ] }, { "ref": "orderly-pricing/1", "stepId": "validate", "activatedAt": "2026-11-06T04:26:22.719Z", "completedAt": "2026-11-06T04:28:27.017Z", "ms": 124298, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "e2b228fc-5960-4805-b75f-72eb90bd79ea" ] }, { "ref": "orderly-pricing/1", "stepId": "evaluate", "activatedAt": "2026-11-06T04:28:27.018Z", "completedAt": "2026-11-06T04:34:56.888Z", "ms": 389870, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "c665e12f-6506-4bdc-b6b3-99cf703f157f" ] }, { "ref": "orderly-pricing/1", "stepId": "finish", "activatedAt": "2026-11-06T04:34:56.889Z", "completedAt": "2026-11-06T04:37:51.273Z", "ms": 174384, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "0ab053b7-768a-4094-a8fc-86d4c7398ba9" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T04:12:58.249Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T04:12:58.143Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 240424.5 }, "sessions": { "total": 7, "taskScoped": 5, "byGezel": { "alvaro": 1, "aysel": 2, "mahnoor": 2, "pavlina": 1, "tamsin": 1 }, "perTask": { "orderly-pricing/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.14501953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.14501953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-developer-craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "scenarioId": "craftbook-bug-fix-tdd", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "reviewedAt": "2026-11-06T05:08:27.151Z", "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 12733, "maxSessionContextFill": 0.14501953125, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.14501953125 }, "nativeFailureLines": [], "sampledPeaks": { "gpuMiB": 10638, "processTreeRssMiB": 14900, "wholeSystemRamMiB": 31043 }, "note": "Explicit CPU offload; shared GPU memory and paging unmeasured. Whole-system RAM includes other applications. Released-slot observations are not continuous KV peaks." }, "status": "scored", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/fix-notes.md", "artifactBytes": 2077, "artifactSha256": "ce4e8e329b8e5cf6bce92be568c6f7db43162f95597dc94823ca8c9a0bf79717", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/bu/bug-fix-tdd/versions/2.0.7/test.json", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ], "scoreAxes": { "Repro-first discipline": 9, "Fix minimality": 9, "Honest verification": 7 }, "axisJustifications": { "Repro-first discipline": "The appended boundary test really fails 80 versus 68 before any source edit. The same command later passes all four tests; the original three tests are preserved verbatim.", "Fix minimality": "The complete source differs only by changing the threshold comparison from > to >=. There is no input special case, unrelated source churn, or weakened original test.", "Honest verification": "The red and green receipts are real, but Files touched omits the modified test. Validation says nothing remains unverified despite the fix notes excluding rounding edge cases; the reviewer misses both inconsistencies." }, "meanScore": 8.33, "justification": "A real failing regression precedes the one-character boundary fix, and the same suite then passes 4/4. Original tests remain intact. The changed-file list omits the regression test, and validation overstates coverage despite an explicit rounding limitation elsewhere. The independent reviewer accepts those reporting inconsistencies.", "reviewBasis": "Read all five complete documents, source, tests and seeded report. Compared source/test contents against frozen fixtures and inspected append, red-run, source-edit and green-run receipts. No additional execution or model retry.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/repro.md", "bytes": 1606, "sha256": "137d52182fb37982d2572fd6f280a45400f40260b2ecf84210ead48c17539d0f", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/diagnosis.md", "bytes": 2825, "sha256": "bd5d7b0d6bb5126125fd7c8c9acf360e7b9e2fb9cdee8e88c1e5ce6833395f18", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/fix-notes.md", "bytes": 2077, "sha256": "ce4e8e329b8e5cf6bce92be568c6f7db43162f95597dc94823ca8c9a0bf79717", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/validation.md", "bytes": 1399, "sha256": "35fce783703120e86ef729f1d7a265a494a255d9d7b2c8ca524ea06fff35eaa8", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/review.md", "bytes": 3281, "sha256": "7a1f49866f59647d139c7ddc81f07683d3a186faf2cb92f186016e879d76efd2", "status": "reviewedSupportingArtifact" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:06.456Z", "reviewId": "077", "cellId": "r2-saluki-cuda-developer-craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "scenarioId": "craftbook-bug-fix-tdd", "status": "scored", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "baselineCellId": "r2-saluki-cuda-developer-craftbook-bug-fix-tdd", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "3ab8eb18ad6363e3ac6c9f67eceaf9c454445a68c001cf7542e307acee676536", "evidenceSha256": "ed2ee69ad44f99e527eecf4893a41b122dbcb8d53b7a4b40dba60618b7e08c94", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/fix-notes.md", "bytes": 2077, "sha256": "ce4e8e329b8e5cf6bce92be568c6f7db43162f95597dc94823ca8c9a0bf79717", "shownBytes": 2077, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/repro.md", "bytes": 1606, "sha256": "137d52182fb37982d2572fd6f280a45400f40260b2ecf84210ead48c17539d0f", "shownBytes": 1606, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/diagnosis.md", "bytes": 2825, "sha256": "bd5d7b0d6bb5126125fd7c8c9acf360e7b9e2fb9cdee8e88c1e5ce6833395f18", "shownBytes": 2825, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/validation.md", "bytes": 1399, "sha256": "35fce783703120e86ef729f1d7a265a494a255d9d7b2c8ca524ea06fff35eaa8", "shownBytes": 1399, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/review.md", "bytes": 3281, "sha256": "7a1f49866f59647d139c7ddc81f07683d3a186faf2cb92f186016e879d76efd2", "shownBytes": 3281, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4\\workspace\\orderly-pricing\\source\\bug-report.md", "bytes": 313, "sha256": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "shownBytes": 313, "truncated": false } ], "scoreAxes": { "Repro-first discipline": 8, "Fix minimality": 8, "Honest verification": 7 }, "justification": "repro.md quotes a real red run (exit 1, 3 pass/1 fail, 80 !== 68) for a test in tests/pricing.test.mjs. The fix is a one-character >→>= at the diagnosed site, and it is not special-cased. Validation shows a green 4/4 run. The review is a thorough PASS but leans on artifacts and inspection.", "limitations": "Source files and the actual diff are not shown. Run outputs are only as quoted in the artifacts. The review.md findings table is empty. The reviewer did not re-run the tests. Revert-bites claim was by inspection only. Validation's \"Nothing remains unverified\" conflicts with fix-notes' unverified float edge cases.", "meanScore": 7.666666666666667, "durationMs": 14992, "costUsd": 0.0361064, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 8050, "cache_read_input_tokens": 824, "output_tokens": 382, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 8050, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 382, "cache_read_input_tokens": 824, "cache_creation_input_tokens": 8050, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 8050 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 382, "cacheReadInputTokens": 824, "cacheCreationInputTokens": 8050, "webSearchRequests": 0, "costUSD": 0.0361064, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 8050, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-bug-fix-tdd / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 25m 8s; budget used: 6.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 105947`; `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1588,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.06695007492844511`; `facts.toolUse.totalToolCalls: 38`; `facts.timing.timeToFirstArtifactMs: 105947`; `facts.outcome.durationMs: 1508262` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14900 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31043 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.91 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 63566 + 10591 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 8 → 6 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 240s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.63 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 105947`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1588,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.06695007492844511`\n- `facts.toolUse.totalToolCalls: 38`\n- `facts.timing.timeToFirstArtifactMs: 105947`\n- `facts.outcome.durationMs: 1508262`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 105947`; `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1588,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.06695007492844511`; `facts.toolUse.totalToolCalls: 38`; `facts.timing.timeToFirstArtifactMs: 105947`; `facts.outcome.durationMs: 1508262`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.5/10** remains unchanged. Advisory **8.33/10**, codex-inline/current-session.\n\n| Axis | Score | Evidence |\n|---|---:|---|\n| Repro-first discipline | 9 | The appended boundary test really fails 80 versus 68 before any source edit. The same command later passes all four tests; the original three tests are preserved verbatim. |\n| Fix minimality | 9 | The complete source differs only by changing the threshold comparison from > to >=. There is no input special case, unrelated source churn, or weakened original test. |\n| Honest verification | 7 | The red and green receipts are real, but Files touched omits the modified test. Validation says nothing remains unverified despite the fix notes excluding rounding edge cases; the reviewer misses both inconsistencies. |\n\nA real failing regression precedes the one-character boundary fix, and the same suite then passes 4/4. Original tests remain intact. The changed-file list omits the regression test, and validation overstates coverage despite an explicit rounding limitation elsewhere. The independent reviewer accepts those reporting inconsistencies.\n\nRead all five complete documents, source, tests and seeded report. Compared source/test contents against frozen fixtures and inspected append, red-run, source-edit and green-run receipts. No additional execution or model retry.\n\n## Positives\n\n- All 28 deterministic checks and six workflow stages complete, with five task-scoped sessions (outcome.success; outcome.reason; continuity.steps.completed=6; continuity.sessions.taskScoped=5).\n- A real failing regression is appended at 04:14:38; the red run at 04:14:45 reports 3 pass / 1 fail with 80 !== 68. The source edit occurs later, at 04:22:49, and the same test command passes 4/4 at 04:27:24 (toolUse.byTool.run_package_script=2; advisory-consistency-audit.json chronology).\n- The source differs from the seeded fixture by only > to >= at the defect site. The original three tests are an unchanged prefix of the final test file; package.json and the bug report are unchanged (artifacts.otherFileCount; fixture audit; source capture).\n- The final notes explicitly identify floating-point edge cases as unverified. The reviewer correctly distinguishes inspecting revert behavior from running a separate mutant check (artifacts.otherFileCount; fix-notes.md Risk; review.md criterion 5).\n\n## Negatives\n\n- Files touched names only src/pricing.js, omitting the appended regression test. Review criterion 6 nevertheless says every touched file is named (toolUse.byTool.append_to_file=1; captured test diff; fix-notes.md and review.md).\n- Validation ends with Nothing remains unverified, contradicting the final notes excluding floating-point edge cases. One qty=20 example does not prove the entire discount branch correct; the primary report overstates that result (artifacts.otherFileCount; validation.md Result; fix-notes.md Risk).\n- A diagnosis write was rejected for uncited line numbers and later repaired. This is a write guard rejection, not a workflow gate rejection; continuity.steps.gateRejections remains zero (toolUse.byTool.write_artifact; advisory-tool-evidence.json).\n- The 38 calls and 25m08s include substantial documentation/review time for a one-character source repair. Fixed efficiency is limited by call count even though the real red-to-green workflow succeeds (toolUse.totalToolCalls=38; outcome.durationMs=1508262; score.axes.efficiency).\n\n## Performance and GPU fit\n\nElapsed **25m8s**, 38 calls, 10591 engine-generated tokens at **8.91 tok/s**, prefill **402.92 tok/s** (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10638/12227 MiB**, process-tree RSS **14900 MiB**, whole-system RAM **31043/31865 MiB**. Whole-system memory includes other applications; shared GPU memory and paging were not measured. No new campaign peak.\n\nOne native 0.1.57 CUDA launch, 65,536-token context, one slot, f16 KV; dense FFN placement is 39/64 layers on GPU and 25 on CPU. This is hybrid residency. Captured session fill peaks at 14.5%; largest observed released native slot 12733 tokens (19.4%). No compaction, native endpoint failure, explicit OOM or context overflow observed. Released-slot observations are not continuous KV peaks.\n\n## Tactical experiments — not applied\n\nIn ../gilde/data/craftbook-templates/bu/bug-fix-tdd/versions/2.0.7/craftbook.json, add to fix.prompt: “Derive Files touched from actual writes or the diff, including regression tests.” Add to validate.prompt and evaluate.prompt: “List verified scope and untested cases separately; reconcile those lists across the final notes and validation. A passing example is not proof of all inputs.” Evidence: toolUse.byTool.append_to_file=1 and the documented coverage contradiction. These are proposed content experiments only; no prompt, sampling, budget or gate changes were applied.\n\n## Strategic experiments — not applied\n\nIn packages/service/src/chat/manager.ts, evaluate receipt-bound changed-file evidence with path and before/after hashes so a reviewer can distinguish inspected files from modified files. In packages/core/src/checks/claims.ts, evaluate structured test-scope claims against actual run receipts, without interpreting a passing suite as exhaustive correctness. Keep red/green failure receipts as positive workflow evidence rather than generic tool failure penalties. Measure omissions, unsupported verification claims and additional tool cost.\n\n## Generalization and next experiment\n\nR1 TDD also scored fixed 8.5 and advisory 8.33 (9/9/7), with a real red-to-green repair and a missing test in Files touched. R2 removes the earlier unsupported summation-style explanation but retains an overbroad coverage statement. This is repetition variability under frozen tuning/native identity. Unrelated A/B reporting and API review likewise overstate evidence, while refactor again misstates touched files. A future receipt-bound reporting experiment should cover all these task types, preserve independent functional outcomes, and measure false claims and review cost; do not encode fixture solutions in a model profile.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/facts.json): pass, fixed 8.5, 23.04 min, 8.58 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/facts.json): fail, fixed 5.5, 44.23 min, 114.86 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-36-26-068Z-suho/facts.json): fail, fixed 5.6, 21.16 min, 119.57 tok/s.\n\nOther-model history is contextual, not a controlled native-version comparison.\n\n## Strategic note\n\nThe functional repair and red-to-green sequence are genuine; reporting consistency remains weaker than the code. The standard harness adds its independent behavior and mutant checks, which are not attributed to the model.\n\nNo source, tuning, gate or execution changes, downloads, added evals or capability retries were performed by the monitor.\n\n## Raw evidence\n\nfacts.json, score.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json, recording/chat-events.jsonl, and captured artifacts/workspace.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-find-vs-create", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-find-vs-create", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T04:40:51.156Z", "finishedAt": "2026-11-06T04:42:47.247Z", "durationMs": 116091, "success": true, "reason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.03864862480461955 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 116091 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" }, "facts": { "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 18, "peakRssMb": 9833, "peakCpuPercent": 1203.5, "firstAtMs": 0, "lastAtMs": 101173 }, "gpu": { "available": true, "sampleCount": 18, "peakUtilPercent": 99, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 27227, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11934, "totalOutputTokens": 535, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T04:42:40.155Z" } }, "source": "engine-log" }, "derived": { "durationMs": 102648, "meanTokensPerSec": 9.34, "genTokensPerSec": 9.34, "genTokens": 535, "promptTokensPerSec": 629.26, "peakRssMb": 9833, "peakGpuUtilPercent": 99, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shop-site-pages/1", "stepId": "scope", "activatedAt": "2026-11-06T04:42:35.781Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T04:40:56.979Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T04:40:56.903Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1510467529296875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1510467529296875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/whats-new/1.26234", "knowledge://bendyline/handboek/craftbook/accessibility-retrofit", "knowledge://bendyline/handboek/craftbook/podcast-chaptering" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "durationMs": 116091, "timeoutMs": 3003755, "budgetUsedFraction": 0.03864862480461955 }, "timing": { "startedAt": "2026-11-06T04:40:51.156Z", "finishedAt": "2026-11-06T04:42:47.247Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 44630, "firstTurnTtftMs": 26595, "timeToFirstToolCallMs": 82286 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "suggest_work": 1, "start_work": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/shop-site-pages/pages/index.html", "finalBytes": 168, "growth": [] }, { "path": "workspace/shop-site-pages/pages/shipping.html", "finalBytes": 165, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 7758, "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } ], "latest": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T04:40:51.159Z] [trial] id=craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe scenario=craftbook-find-vs-create model=underdog-saluki-27b-iq2-mix", "[2026-11-06T04:40:51.160Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T04:40:51.160Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-find-vs-create-KIH49u", "[2026-11-06T04:40:51.159Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "[2026-11-06T04:40:51.160Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T04:40:51.162Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T04:40:51.167Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T04:40:51.173Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-find-vs-create-KIH49u", "[2026-11-06T04:40:51.173Z] [trial] throughput-scaled ceiling: 20m → 50m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T04:40:56.906Z] [trial] daemon spawned pid=29252 port=https://127.0.0.1:62185", "[2026-11-06T04:40:58.322Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T04:40:58.338Z] [trial] meester=tamsin", "[2026-11-06T04:40:58.338Z] [trial] running scenario setup hook", "[2026-11-06T04:40:58.659Z] [trial] scenario setup complete", "[2026-11-06T04:40:58.661Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T04:40:58.741Z] [poll] initial digests hard=c03c50d0941e06b0 soft=5c08e086cad82b7e", "[2026-11-06T04:40:58.997Z] [poll] hard-progress: c03c50d0941e06b0 → 2348ce005556254a after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/3B)", "[2026-11-06T04:42:16.105Z] [poll] hard-progress: 2348ce005556254a → a42208caeb2cf4db after 77s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=124,stream=412) sniff=1/3B)", "[2026-11-06T04:42:39.678Z] [poll] terminal: success=true reason=task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "[2026-11-06T04:42:40.187Z] [capture] dumped 1 session(s)", "[2026-11-06T04:42:40.191Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-06T04:42:40.257Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-06T04:42:47.246Z] [trial] daemon shut down", "[2026-11-06T04:42:47.255Z] [trial] PASS duration=116091ms reason=task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored" ] }, "metrics": { "process": { "sampleCount": 18, "peakRssMb": 9833, "peakCpuPercent": 1203.5, "firstAtMs": 0, "lastAtMs": 101173 }, "gpu": { "available": true, "sampleCount": 18, "peakUtilPercent": 99, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 18, "peakUsedMb": 27227, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11934, "totalOutputTokens": 535, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T04:42:40.155Z" } }, "source": "engine-log" }, "derived": { "durationMs": 102648, "meanTokensPerSec": 9.34, "genTokensPerSec": 9.34, "genTokens": 535, "promptTokensPerSec": 629.26, "peakRssMb": 9833, "peakGpuUtilPercent": 99, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T04:40:51.156Z", "finishedAt": "2026-11-06T04:42:47.247Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 44630, "firstTurnTtftMs": 26595, "timeToFirstToolCallMs": 82286 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shop-site-pages/1", "stepId": "scope", "activatedAt": "2026-11-06T04:42:35.781Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T04:40:56.979Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T04:40:56.903Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1510467529296875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1510467529296875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "scenarioId": "craftbook-find-vs-create", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "reviewedAt": "2026-11-06T05:08:27.221Z", "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 12221, "maxSessionContextFill": 0.1510467529296875, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1510467529296875 }, "nativeFailureLines": [], "sampledPeaks": { "gpuMiB": 10628, "processTreeRssMiB": 9833, "wholeSystemRamMiB": 27227 }, "note": "Explicit CPU offload; shared GPU memory and paging unmeasured. Whole-system RAM includes other applications. Released-slot observations are not continuous KV peaks." }, "status": "noArtifact", "artifactPath": null, "artifactBytes": 0, "artifactSha256": null, "axisSource": "evals/src/craftbooks/authoring/find-vs-create.ts", "axes": [], "scoreAxes": {}, "axisJustifications": {}, "meanScore": null, "justification": "The two calls find and invoke the existing SEO recipe without authoring another. The scenario ends at task creation; neither seeded page was edited and no deliverable was produced. This pass demonstrates recipe selection and launch, not completed SEO work.", "reviewBasis": "Read the scenario and both tool receipts; compared both captured HTML inputs byte-for-byte to source fixtures. Verified zero completed workflow stages and zero task-scoped sessions. No visual or artifact score is assigned to seeded inputs.", "deliverables": [], "excludedSeededInputs": [ { "path": "pages/index.html", "unchanged": true, "bytes": 168, "sha256": "247f7de92e8ca6a36b9e7c485e7b7649efd85739ef211faa04c946324f284fb8", "status": "seededInputExcluded" }, { "path": "pages/shipping.html", "unchanged": true, "bytes": 165, "sha256": "1ce475ec4ca411c79d190c428b288947732ec62344799d746155e08ac208ef8b", "status": "seededInputExcluded" } ] }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:09.416Z", "reviewId": "078", "cellId": "r2-saluki-cuda-complex-work-craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "scenarioId": "craftbook-find-vs-create", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-find-vs-create", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "76df4f910632107e5762f93ae7182f34d27e2c7cda3c3378727e8f725fe9846a", "evidenceSha256": "addec914e9a6117ce497e225f8e88c346e86e3dad6049b4eefc23358009af0cc", "artifactFiles": [], "contextFiles": [], "scoreAxes": {}, "justification": "No deliverable to score: artifacts is empty, declaredAxes is empty, and missingDeclaredPaths is empty. The evidence has no transcript or tool-call trace, so it doesn't show whether suggest_craftbook or invoke_craftbook was called, or why no output exists. The run is marked runner-terminal.", "limitations": "Source-only review with no runtime execution. There is no evidence of the agent's actions or of the workspace's final state. The cause of the missing output can't be determined, so this is neither a model success nor a model failure. No numeric scores are given.", "meanScore": null, "durationMs": 14307, "costUsd": 0.009052000000000001, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1482, "cache_read_input_tokens": 700, "output_tokens": 305, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1482, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 305, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1482, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1482 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 305, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1482, "webSearchRequests": 0, "costUSD": 0.009052000000000001, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1482, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-find-vs-create / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.8 / 10** (band: ship-ready)\n**Outcome: success** (duration: 1m 56s; budget used: 3.9%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 5 / 10 | The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.03864862480461955`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 116091` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **8.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9833 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 99% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27227 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.34 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 11934 + 535 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 50% of its measured signals.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.03864862480461955`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 116091`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.03864862480461955`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 116091`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.8/10** remains unchanged. Advisory **noArtifact; no declared axes**, codex-inline/current-session.\n\nThe two calls find and invoke the existing SEO recipe without authoring another. The scenario ends at task creation; neither seeded page was edited and no deliverable was produced. This pass demonstrates recipe selection and launch, not completed SEO work.\n\nRead the scenario and both tool receipts; compared both captured HTML inputs byte-for-byte to source fixtures. Verified zero completed workflow stages and zero task-scoped sessions. No visual or artifact score is assigned to seeded inputs.\n\n## Positives\n\n- suggest_work returns the existing SEO Metadata Pack recipe, and start_work invokes exactly that option in the correct project. No new local recipe is authored (toolUse.totalToolCalls=2; outcome.success; outcome.reason; advisory-tool-evidence.json).\n- The task contains the four expected stages and starts at scope. Both captured HTML files remain byte-identical to the fixture, consistent with the instruction to find and start the recipe (continuity.steps.perStep; fixture audit).\n- Routing is quick: 1m56s elapsed, two calls, 535 engine-generated tokens, with no interventions or red flags (outcome.durationMs; toolUse.totalToolCalls; perf.derived; autoAnswer.total; toolUse.redFlags).\n\n## Negatives\n\n- No SEO work is completed: there are zero completed stages and zero task-scoped sessions before the selection probe ends. The successful result must not be reported as completed metadata edits (continuity.steps.completed=0; continuity.sessions.taskScoped=0; unchanged seeded pages).\n- The latest sniff still reports 1/2 and no matching task even though the terminal verdict finds the task. Fixed quality=5 and composite=8.8 retain this stale snapshot; they are not evidence of a half-completed SEO deliverable (sniff.latest; outcome.reason; score.axes.quality).\n\n## Performance and GPU fit\n\nElapsed **1m56s**, 2 calls, 535 engine-generated tokens at **9.34 tok/s**, prefill **629.26 tok/s** (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10628/12227 MiB**, process-tree RSS **9833 MiB**, whole-system RAM **27227/31865 MiB**. Whole-system memory includes other applications; shared GPU memory and paging were not measured. No new campaign peak.\n\nOne native 0.1.57 CUDA launch, 65,536-token context, one slot, f16 KV; dense FFN placement is 39/64 layers on GPU and 25 on CPU. This is hybrid residency. Captured session fill peaks at 15.1%; largest observed released native slot 12221 tokens (18.6%). No compaction, native endpoint failure, explicit OOM or context overflow observed. Released-slot observations are not continuous KV peaks.\n\n## Tactical experiments — not applied\n\nIn evals/src/craftbooks/authoring/find-vs-create.ts, a future prompt-maintenance experiment could replace the legacy kickoff tool names suggest_craftbook/invoke_craftbook with the available suggest_work/start_work surface, while preserving the selection objective and expected outcome. The model successfully uses the current tools despite the legacy wording (toolUse.byTool; two recorded receipts). Do not change this campaign or add a task-specific instruction to the model profile.\n\n## Strategic experiments — not applied\n\nIn evals/src/craftbooks/authoring/find-vs-create.ts successCheck, a separately versioned reporting change should emit the final 2/2 selection evidence when the bundled task exists, before returning the terminal verdict. Keep routing success and execution completion as distinct result fields in the campaign report, and continue excluding seeded HTML in evals/src/trial-llm-judge.ts. This addresses the stale sniff/terminal mismatch without claiming the recipe itself ran.\n\n## Generalization and next experiment\n\nR1 also passed fixed 8.8 with two routing calls and no produced artifact. R2 reproduces selection behavior, not downstream SEO competence. The unrelated developer-start-work probe likewise ends at routing, while TDD completes six stages with real test receipts. Test final-verdict evidence synchronization across both routing probes and a file-producing workflow before adopting any reporting change; do not merge routing-only passes into a claim that the resulting work was completed.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/facts.json): pass, fixed 8.8, 1.92 min, 8.12 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/facts.json): pass, fixed 8.8, 0.82 min, 108.41 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-43-36-219Z-ykgt/facts.json): pass, fixed 8.8, 0.80 min, 104.14 tok/s.\n\nOther-model history is contextual, not a controlled native-version comparison.\n\n## Strategic note\n\nSuccessful recipe selection is a useful narrow outcome. Neither model loading nor routing a task establishes end-to-end competence at the work being routed.\n\nNo source, tuning, gate or execution changes, downloads, added evals or capability retries were performed by the monitor.\n\n## Raw evidence\n\nfacts.json, score.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json, recording/chat-events.jsonl, and captured artifacts/workspace.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-edit-midtask", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-edit-midtask", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T04:45:07.149Z", "finishedAt": "2026-11-06T05:48:47.991Z", "durationMs": 3820842, "success": true, "reason": "diagnosed the unwinnable gate, rewrote the failing step prompt, grew the recipe with a verification step, and completed the task", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2899315 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-edit-midtask", "bytes": 594, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.5653431083586055 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 55 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2899315 }, { "source": "facts", "path": "outcome.durationMs", "value": 3820842 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 640, "peakRssMb": 14431, "peakCpuPercent": 4311.5, "firstAtMs": 0, "lastAtMs": 3807579 }, "gpu": { "available": true, "sampleCount": 640, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 640, "peakUsedMb": 31288, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 112859, "totalOutputTokens": 28514, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 196173, "todayTokensCached": 181452, "todayTokensOut": 2996, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 196173, "totalTokensCached": 181452, "totalTokensOut": 2996, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 12 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7973, "cachedInputTokens": 7813, "outputTokens": 108, "cost": 0, "durationMs": 214306, "outputTokensPerSec": 8.71228800642006, "at": "2026-11-06T05:48:38.996Z" }, "lastUpdated": "2026-11-06T05:48:38.996Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T05:48:42.011Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3809236, "meanTokensPerSec": 8.48, "genTokensPerSec": 8.48, "genTokens": 28514, "promptTokensPerSec": 360.93, "peakRssMb": 14431, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 196173, "output": 2996 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "quarterly-brief/1", "stepId": "draft", "activatedAt": "2026-11-06T04:45:13.980Z", "completedAt": "2026-11-06T05:34:48.756Z", "ms": 2974776, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "quarterly-brief/1", "stepId": "polish", "activatedAt": "2026-11-06T05:34:48.758Z", "completedAt": "2026-11-06T05:43:57.752Z", "ms": 548994, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "a77d5863-2df3-473b-926e-9b070f326439" ] }, { "ref": "quarterly-brief/1", "stepId": "verify", "activatedAt": "2026-11-06T05:43:57.753Z", "completedAt": "2026-11-06T05:48:38.918Z", "ms": 281165, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "a77d5863-2df3-473b-926e-9b070f326439" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T04:45:12.910Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T04:45:12.817Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 548994 }, "sessions": { "total": 6, "taskScoped": 1, "byGezel": { "hayoung": 1, "lena": 1, "reza": 2, "reza-2": 1, "tyra": 1 }, "perTask": { "quarterly-brief/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.2, "maxMessages": 10, "maxContextFill": 0.33209228515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.33209228515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "diagnosed the unwinnable gate, rewrote the failing step prompt, grew the recipe with a verification step, and completed the task", "durationMs": 3820842, "timeoutMs": 6758448, "budgetUsedFraction": 0.5653431083586055 }, "timing": { "startedAt": "2026-11-06T04:45:07.149Z", "finishedAt": "2026-11-06T05:48:47.991Z", "timeToFirstArtifactMs": 2899315, "timeToLastArtifactWriteMs": 3617128, "timeToFirstTokenMs": 45003, "firstTurnTtftMs": 35390, "timeToFirstToolCallMs": 54392 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 55, "byTool": { "get_task": 6, "list_tasks": 1, "craftbook_read": 1, "list_dir": 3, "read_file": 13, "read_task_notes": 4, "craftbook_write": 1, "find_files": 4, "message_gezel": 3, "list_artifacts": 1, "write_task_note": 3, "delegate_developer": 1, "advance_task_step": 5, "delegate_writer": 1, "list_gezels": 1, "stat": 4, "write_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 3 }, "sniff": { "progression": [ { "atMs": 7156, "filePath": "craftbook-edit-midtask", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" }, { "atMs": 1426993, "filePath": "craftbook-edit-midtask", "bytes": 0, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 2899784, "filePath": "craftbook-edit-midtask", "bytes": 500, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3042110, "filePath": "craftbook-edit-midtask", "bytes": 594, "score": 3, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3814328, "filePath": "craftbook-edit-midtask", "bytes": 594, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-edit-midtask", "bytes": 594, "score": 4, "scoreMax": 4, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T04:45:07.153Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T04:45:07.153Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-edit-midtask-caPll9", "[2026-11-06T04:45:07.153Z] [trial] id=craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas scenario=craftbook-edit-midtask model=underdog-saluki-27b-iq2-mix", "[2026-11-06T04:45:07.153Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "[2026-11-06T04:45:07.153Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T04:45:07.155Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T04:45:07.159Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T04:45:07.164Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-edit-midtask-caPll9", "[2026-11-06T04:45:07.164Z] [trial] throughput-scaled ceiling: 45m → 113m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T04:45:12.853Z] [trial] daemon spawned pid=17632 port=https://127.0.0.1:61599", "[2026-11-06T04:45:13.737Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T04:45:13.754Z] [trial] meester=imara", "[2026-11-06T04:45:13.754Z] [trial] running scenario setup hook", "[2026-11-06T04:45:14.208Z] [trial] scenario setup complete", "[2026-11-06T04:45:14.209Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T04:45:14.285Z] [poll] initial digests hard=446313e94534b5d9 soft=6e67de9f5ed8ff01", "[2026-11-06T04:45:14.452Z] [poll] hard-progress: 446313e94534b5d9 → 5fd7833f90d7352f after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-06T04:46:07.055Z] [poll] hard-progress: 5fd7833f90d7352f → cbb00d07b5c4cf69 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=39,stream=76) sniff=1/0B)", "[2026-11-06T04:47:17.965Z] [poll] hard-progress: cbb00d07b5c4cf69 → 4a691e22d1201a4b after 71s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=210,stream=584) sniff=1/0B)", "[2026-11-06T04:50:26.856Z] [poll] hard-progress: 4a691e22d1201a4b → 12fd7190b59f576d after 189s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=742,stream=2168) sniff=1/0B)", "[2026-11-06T04:58:42.350Z] [poll] hard-progress: 12fd7190b59f576d → a714608b36300b65 after 495s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=2101,stream=6179) sniff=1/0B)", "[2026-11-06T05:08:54.270Z] [poll] hard-progress: a714608b36300b65 → 80a697f4c827771c after 612s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=3794,stream=11829) sniff=3/0B)", "[2026-11-06T05:10:00.379Z] [poll] hard-progress: 80a697f4c827771c → 47336a3b0c240913 after 66s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=3957,stream=12313) sniff=3/0B)", "[2026-11-06T05:10:42.093Z] [poll] hard-progress: 47336a3b0c240913 → a3c89622416256c6 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=12,slot=4056,stream=12606) sniff=3/0B)", "[2026-11-06T05:11:17.842Z] [poll] hard-progress: a3c89622416256c6 → 67fe84102ccd8ed6 after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=13,slot=4141,stream=12983) sniff=3/0B)" ] }, "metrics": { "process": { "sampleCount": 640, "peakRssMb": 14431, "peakCpuPercent": 4311.5, "firstAtMs": 0, "lastAtMs": 3807579 }, "gpu": { "available": true, "sampleCount": 640, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 640, "peakUsedMb": 31288, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 112859, "totalOutputTokens": 28514, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 12, "todayTokensIn": 196173, "todayTokensCached": 181452, "todayTokensOut": 2996, "todayCost": 0, "totalTurns": 12, "totalTokensIn": 196173, "totalTokensCached": 181452, "totalTokensOut": 2996, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 12 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7973, "cachedInputTokens": 7813, "outputTokens": 108, "cost": 0, "durationMs": 214306, "outputTokensPerSec": 8.71228800642006, "at": "2026-11-06T05:48:38.996Z" }, "lastUpdated": "2026-11-06T05:48:38.996Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T05:48:42.011Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3809236, "meanTokensPerSec": 8.48, "genTokensPerSec": 8.48, "genTokens": 28514, "promptTokensPerSec": 360.93, "peakRssMb": 14431, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 196173, "output": 2996 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T04:45:07.149Z", "finishedAt": "2026-11-06T05:48:47.991Z", "timeToFirstArtifactMs": 2899315, "timeToLastArtifactWriteMs": 3617128, "timeToFirstTokenMs": 45003, "firstTurnTtftMs": 35390, "timeToFirstToolCallMs": 54392 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "quarterly-brief/1", "stepId": "draft", "activatedAt": "2026-11-06T04:45:13.980Z", "completedAt": "2026-11-06T05:34:48.756Z", "ms": 2974776, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "quarterly-brief/1", "stepId": "polish", "activatedAt": "2026-11-06T05:34:48.758Z", "completedAt": "2026-11-06T05:43:57.752Z", "ms": 548994, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "a77d5863-2df3-473b-926e-9b070f326439" ] }, { "ref": "quarterly-brief/1", "stepId": "verify", "activatedAt": "2026-11-06T05:43:57.753Z", "completedAt": "2026-11-06T05:48:38.918Z", "ms": 281165, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "a77d5863-2df3-473b-926e-9b070f326439" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T04:45:12.910Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T04:45:12.817Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 548994 }, "sessions": { "total": 6, "taskScoped": 1, "byGezel": { "hayoung": 1, "lena": 1, "reza": 2, "reza-2": 1, "tyra": 1 }, "perTask": { "quarterly-brief/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.2, "maxMessages": 10, "maxContextFill": 0.33209228515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.33209228515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "scenarioId": "craftbook-edit-midtask", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-06T06:05:10.636Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/workspace/quarterly-brief/out/summary.md", "artifactBytes": 598, "artifactSha256": "92ee2217ecbc95d317ba6ab358968b47e351c1187a69563183a2a6bfba690e84", "axisSource": "evals/src/craftbooks/authoring/edit-midtask.ts", "axes": [], "scoreAxes": {}, "axisJustifications": {}, "meanScore": null, "justification": "No declared axes. The final brief preserves all source facts and cites their source. The live recipe gains the required prompt and a terminal verification step while retaining its original gates. All three stages complete. Duplicate workers, parked handoffs and redundant writes/transitions make this a slow 64-minute success.", "reviewBasis": "Read the complete brief, seeded notes, initial and final task recipes, all three write contents, handoff receipts and five step-advance calls. Compared original gates structurally and every source bullet to the final brief. No new execution, repair or retry.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/workspace/quarterly-brief/out/summary.md", "bytes": 598, "sha256": "92ee2217ecbc95d317ba6ab358968b47e351c1187a69563183a2a6bfba690e84", "status": "reviewedNoDeclaredAxes" } ], "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 28350, "maxSessionContextFill": 0.33209228515625, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.33209228515625 }, "nativeFailureLines": [], "sampledPeaks": { "gpuMiB": 10646, "processTreeRssMiB": 14431, "wholeSystemRamMiB": 31288 }, "note": "Hybrid CPU/GPU residency. Shared GPU memory and paging unmeasured. Whole-system RAM includes other applications. Native slot observations are not continuous peaks." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:20.848Z", "reviewId": "079", "cellId": "r2-saluki-cuda-complex-work-craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "scenarioId": "craftbook-edit-midtask", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-edit-midtask", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "5163c1fb5009d9c76556ff5de9b48a4715479e5a80cc4989d4e9da9f104a9fe2", "evidenceSha256": "d85a9b0b038250540f782036aca801afce1768f1610d170a2fe5e30c29d1e810", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/workspace/quarterly-brief/out/summary.md", "bytes": 598, "sha256": "92ee2217ecbc95d317ba6ab358968b47e351c1187a69563183a2a6bfba690e84", "shownBytes": 598, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas\\workspace\\quarterly-brief\\data\\notes.txt", "bytes": 544, "sha256": "949c911a4bbdf1083c68bbe94d539edda30128a353f4028a07a65bb717752a47", "shownBytes": 544, "truncated": false } ], "scoreAxes": {}, "justification": "No numeric axes declared. The output summary.md has Wins/Misses/Numbers and a Source Register, matching notes.txt faithfully (minor \"The\" dropped). That suggests the final gate content was met. No craftbook edits or tool calls are shown, so the recipe fix can't be verified.", "limitations": "Evidence shows only the output file and source notes. The craftbook, gate definition, step prompts, added verification step and task completion status are not shown. Execution ended runner-terminal, so no outcome is inferred beyond the artifact's content.", "meanScore": null, "durationMs": 14318, "costUsd": 0.01209, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2259, "cache_read_input_tokens": 700, "output_tokens": 298, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2259, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 298, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2259, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2259 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 298, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2259, "webSearchRequests": 0, "costUSD": 0.01209, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2259, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-edit-midtask / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 63m 41s; budget used: 56.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 2899315`; `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":594,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.5653431083586055`; `facts.toolUse.totalToolCalls: 55`; `facts.timing.timeToFirstArtifactMs: 2899315`; `facts.outcome.durationMs: 3820842` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14431 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31288 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.48 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 112859 + 28514 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 549s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.20 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 33% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 2899315`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":594,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.5653431083586055`\n- `facts.toolUse.totalToolCalls: 55`\n- `facts.timing.timeToFirstArtifactMs: 2899315`\n- `facts.outcome.durationMs: 3820842`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 2899315`; `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":594,\"score\":4,\"scoreMax\":4,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.5653431083586055`; `facts.toolUse.totalToolCalls: 55`; `facts.timing.timeToFirstArtifactMs: 2899315`; `facts.outcome.durationMs: 3820842`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.5/10**, 4/4 checks, unchanged. No declared qualitative axes; advisory **notApplicable**, with the produced brief inspected rather than assigned invented scores.\n\nNo declared axes. The final brief preserves all source facts and cites their source. The live recipe gains the required prompt and a terminal verification step while retaining its original gates. All three stages complete. Duplicate workers, parked handoffs and redundant writes/transitions make this a slow 64-minute success.\n\nRead the complete brief, seeded notes, initial and final task recipes, all three write contents, handoff receipts and five step-advance calls. Compared original gates structurally and every source bullet to the final brief. No new execution, repair or retry.\n\n## Positives\n\n- The original task reaches complete with draft, polish and verify each completed. Both original gate objects are unchanged, the missing heading is now required by the polish prompt, and the new terminal verification repeats the final checks (outcome.success; sniff.latest.score=4; continuity.steps.completed=3; advisory-consistency-audit.json).\n- All eight source bullets survive in the final brief, including the client names, dates, financial figures, delay and stockout estimate. Source Register accurately points to the provided notes; no additional business claims or invented details appear (artifacts.otherFileCount; source/final comparison).\n- Every captured draft is grounded in the notes. The final note cites an actual 598-byte stat receipt and heading read before completing verify (toolUse.byTool.write_file=3; toolUse.byTool.stat=4; recording/task-notes.json).\n\n## Negatives\n\n- The setup calls ensureAuthoringWorker twice, creating reza as task owner and reza-2 as kickoff worker. The helper only reuses an existing worker if creation throws. The duplicate names lead to a failed self-message and extra routing (toolUse.byTool.message_gezel=3; team; initial/final task; recorded roster; helpers.ts).\n- A message to the owner is accepted at 05:11:15 with END YOUR TURN NOW, but the sender continues querying task/files/notes. Several handoffs are parked behind the sender; later both developer and writer are delegated for the same brief (toolUse.byTool.delegate_developer=1; delegate_writer=1; advisory-tool-evidence.json).\n- There are five advance calls for three stages: draft and polish are each advanced twice. The final two summary writes are byte-identical; the latter comes from redundant delegated work. More recipients remain active and are stopped during normal success cleanup (toolUse.byTool.advance_task_step=5; write_file=3; continuity.steps.completed=3; audit).\n- Fifty-five calls, 28,514 engine-generated tokens and 63m40s are expensive for a short grounded brief and one recipe repair. The draft stage alone takes about 49m35s; the framework setup and coordination contribute to this, so do not attribute all latency to model speed (outcome.durationMs; perf.derived; continuity.steps.perStep).\n\n## Performance and GPU fit\n\nElapsed **63m40s**, **55 calls**, **28,514 generated tokens at 8.48 tok/s**, prefill **360.93 tok/s** (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10,646/12,227 MiB**, process-tree RSS **14,431 MiB**, whole-system RAM **31,288/31,865 MiB**. Whole-system RAM includes other applications. Shared GPU memory and paging are unmeasured. No new campaign peaks.\n\nOne native 0.1.57 CUDA launch, 65,536-token context, one slot, f16 KV. Dense FFN placement is 39/64 layers on GPU and 25 on CPU: hybrid residency, not full GPU. Maximum recorded session fill is 33.2%; largest observed released native slot is **28350 tokens (43.3%)**. No compaction, native endpoint failure, explicit OOM or context overflow observed. The two recipient-stop errors follow the successful 4/4 verdict during trial cleanup; they are not native failures.\n\n## Tactical experiments — not applied\n\nIn evals/src/craftbooks/authoring/edit-midtask.ts setup, resolve one workerId once and pass that same value to createTask.assignee and sendWorkerKickoff. In evals/src/craftbooks/authoring/helpers.ts ensureAuthoringWorker, look up the intended existing worker before creating another, or cache the id per setup. This is a concrete harness correction supported by the two created ids and failed self-message, not a prompt trick. Change it only in a future separately identified run.\n\nIn helpers.ts AUTHORING_WORKER_ABOUT, a future general coordination nudge could say: “After a successful asynchronous handoff, end the current turn. Do not poll for the recipient's output while holding its slot; resume when the reply arrives.” The tool already says this, so measure whether a prompt change adds anything before retaining it (toolUse.byTool.message_gezel=3; parked receipts). No sampling or reasoning-budget change is justified by this trial alone.\n\n## Strategic experiments — not applied\n\nIn packages/service/src/chat/manager.ts, extend the existing completedAsyncHandoff handling to test yielding immediately after a successful asynchronous handoff when its recipient is blocked on the sender, rather than waiting for the sender's subsequent stall/final content. Preserve multi-call semantics; measure dispatch delay and task outcomes on unrelated workflows. Bind duplicate handoffs to a task/deliverable identity to avoid two specialists rewriting the same artifact.\n\nMake repeated advance_task_step responses clearly identify an already-completed step instead of reporting another new completion. Keep mutation and gate history idempotent. The recorded three real completion events versus five successful call responses provide a precise regression case; do not loosen gates.\n\n## Generalization and next experiment\n\nR1 also passed fixed 8.5 with a grounded final brief and intact gates, but had five writes including fabricated draft content and a stale overwrite. R2 has three grounded writes, including one redundant identical rewrite. Duplicate-worker setup remains in both. This is variation under unchanged tuning/native identity, not a measured improvement.\n\nUnrelated TDD and refactor demonstrate actual multistage completion with evidence; the selection-only SEO trial demonstrates just routing. A future isolated-worker/explicit-yield experiment should include edit-midtask, invoice fanout and one development workflow, measuring time to first artifact, parked duration, duplicate writes and final gate preservation. Do not generalize a routing pass to task completion.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/facts.json): pass, fixed 8.5, 78.96 min, 8.35 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/facts.json): fail, fixed 4.5, 7.39 min, 111.83 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-47-56-816Z-ngig/facts.json): pass, fixed 8.5, 2.87 min, 106.42 tok/s.\n\nOther-model historical data is contextual, not a controlled native-version comparison.\n\n## Strategic note\n\nThe model did repair the live recipe without weakening its gate, and the final brief is faithful to its source. The largest weakness here is coordination cost with a demonstrable setup defect. No source, prompts, gates, tuning or campaign execution were changed, and no additional model run was performed.\n\n## Raw evidence\n\nfacts.json, score.json, state.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json, recording/task-notes.json and captured workspace.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-invoice-run", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-invoice-run", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T05:51:45.283Z", "finishedAt": "2026-11-06T06:20:11.441Z", "durationMs": 1706158, "success": true, "reason": "craftbook-invoice-run passed 10 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 357988 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-invoice-run", "bytes": 809, "score": 10, "scoreMax": 10, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2272033656009472 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 41 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 357988 }, { "source": "facts", "path": "outcome.durationMs", "value": 1706158 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 286, "peakRssMb": 14508, "peakCpuPercent": 1205.8, "firstAtMs": 0, "lastAtMs": 1693598 }, "gpu": { "available": true, "sampleCount": 286, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 286, "peakUsedMb": 30827, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 97950, "totalOutputTokens": 11613, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 84680, "todayTokensCached": 75218, "todayTokensOut": 2086, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 84680, "totalTokensCached": 75218, "totalTokensOut": 2086, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7824, "cachedInputTokens": 7491, "outputTokens": 122, "cost": 0, "durationMs": 92399, "outputTokensPerSec": 9.253297156255305, "at": "2026-11-06T06:20:02.758Z" }, "lastUpdated": "2026-11-06T06:20:02.758Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T06:20:06.457Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1694936, "meanTokensPerSec": 9.04, "genTokensPerSec": 9.04, "genTokens": 11613, "promptTokensPerSec": 387.29, "peakRssMb": 14508, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 84680, "output": 2086 } } } }, "craftbook": { "craftbookId": "invoice-run", "tasksObserved": 6, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 10, "activationEvents": 4, "completed": 8, "gateApprovals": 7, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "fieldnote-studio-office/1", "stepId": "scope", "activatedAt": "2026-11-06T05:51:52.809Z", "completedAt": "2026-11-06T05:57:43.367Z", "ms": 350558, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "9822a76d-4314-4ce5-a43c-72900cd22be9" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "draft", "activatedAt": "2026-11-06T05:57:43.368Z", "completedAt": "2026-11-06T05:57:43.711Z", "ms": 343, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "fieldnote-studio-office/1", "stepId": "collect", "activatedAt": "2026-11-06T05:57:43.712Z", "completedAt": "2026-11-06T06:13:21.282Z", "ms": 937570, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "06a05a54-5901-497d-9612-501429d8a317" ] }, { "ref": "fieldnote-studio-office/2", "stepId": "draft-invoice", "activatedAt": "2026-11-06T05:57:43.387Z", "completedAt": "2026-11-06T06:01:36.784Z", "ms": 233397, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "cea86a85-bb01-4299-89bd-493f2eef50e4" ] }, { "ref": "fieldnote-studio-office/3", "stepId": "draft-invoice", "activatedAt": "2026-11-06T05:57:43.497Z", "completedAt": "2026-11-06T06:05:39.895Z", "ms": 476398, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "6a837033-03f0-4d61-ad78-201ebe19a388" ] }, { "ref": "fieldnote-studio-office/4", "stepId": "draft-invoice", "activatedAt": "2026-11-06T05:57:43.540Z", "completedAt": "2026-11-06T06:09:39.413Z", "ms": 715873, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "4eeab0df-dc93-483a-a3eb-02adbc8f4ca7" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "evaluate", "activatedAt": "2026-11-06T06:13:21.283Z", "completedAt": "2026-11-06T06:17:26.494Z", "ms": 245211, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "9afc7f47-464f-478c-a3f3-19c82f75008d" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "finish", "activatedAt": "2026-11-06T06:17:26.495Z", "completedAt": "2026-11-06T06:20:02.701Z", "ms": 156206, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "791720fc-5d74-4a53-a9ef-e785778273ba" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T05:51:51.279Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T05:51:51.166Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 297884.5 }, "sessions": { "total": 9, "taskScoped": 7, "byGezel": { "ai": 1, "aurelien": 1, "birna": 1, "corina": 3, "mahi": 1, "sulien": 1, "taeyang": 1 }, "perTask": { "fieldnote-studio-office/1": 4, "fieldnote-studio-office/4": 1, "fieldnote-studio-office/3": 1, "fieldnote-studio-office/2": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7, "maxMessages": 2, "maxContextFill": 0.1172027587890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1172027587890625 }, "fanout": { "hosts": 1, "childrenSpawned": 3, "childrenCompleted": 3, "childrenFailed": 0, "barrierHolds": 1, "barrierReleases": 1, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 6, "tasksWithReferences": 4, "items": 20, "citations": [ "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-invoice-run passed 10 deterministic craftbook checks", "durationMs": 1706158, "timeoutMs": 7509387, "budgetUsedFraction": 0.2272033656009472 }, "timing": { "startedAt": "2026-11-06T05:51:45.283Z", "finishedAt": "2026-11-06T06:20:11.441Z", "timeToFirstArtifactMs": 357988, "timeToLastArtifactWriteMs": 1541038, "timeToFirstTokenMs": 35239, "firstTurnTtftMs": 21330, "timeToFirstToolCallMs": 41639 }, "team": { "totalGezelsCreated": 10, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Office manager", "Office Administrator", "Designer", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 41, "byTool": { "read_files": 1, "write_artifact": 3, "write_task_note": 3, "read_task_notes": 3, "read_file": 16, "write_file": 5, "advance_task_step": 5, "read_artifact": 4, "list_dir": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/fieldnote-studio-office/invoices/2026-042.html", "finalBytes": 4560, "growth": [] }, { "path": "workspace/fieldnote-studio-office/invoices/2026-043.html", "finalBytes": 4547, "growth": [] }, { "path": "workspace/fieldnote-studio-office/invoices/2026-044.html", "finalBytes": 4530, "growth": [] } ], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 7775, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 1, "scoreMax": 10, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: clients.json, ledger.json" }, { "atMs": 42317, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 588392, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 828436, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 1067731, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 3, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 1275601, "filePath": "craftbook-invoice-run", "bytes": 809, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 1700681, "filePath": "craftbook-invoice-run", "bytes": 809, "score": 10, "scoreMax": 10, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-invoice-run", "bytes": 809, "score": 10, "scoreMax": 10, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T05:51:45.287Z] [trial] id=craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw scenario=craftbook-invoice-run model=underdog-saluki-27b-iq2-mix", "[2026-11-06T05:51:45.287Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "[2026-11-06T05:51:45.287Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T05:51:45.287Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-invoice-run-EINTMf", "[2026-11-06T05:51:45.287Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T05:51:45.290Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T05:51:45.294Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T05:51:45.300Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-invoice-run-EINTMf", "[2026-11-06T05:51:45.300Z] [trial] throughput-scaled ceiling: 50m → 125m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T05:51:51.224Z] [trial] daemon spawned pid=47700 port=https://127.0.0.1:51891", "[2026-11-06T05:51:52.368Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T05:51:52.389Z] [trial] running scenario setup hook", "[2026-11-06T05:51:52.389Z] [trial] meester=sulien", "[2026-11-06T05:51:52.965Z] [trial] scenario setup complete", "[2026-11-06T05:51:52.966Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T05:51:53.009Z] [poll] initial digests hard=047c7921b6b3a446 soft=f653044c31efaef1", "[2026-11-06T05:51:53.093Z] [poll] hard-progress: 047c7921b6b3a446 → acfa9d3478f45d12 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-06T05:51:58.209Z] [poll] hard-progress: acfa9d3478f45d12 → 83a26fcf50f58be6 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-06T05:52:27.652Z] [poll] hard-progress: 83a26fcf50f58be6 → cdee596cf208588e after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=23,stream=49) sniff=2/0B)", "[2026-11-06T05:56:29.589Z] [poll] hard-progress: cdee596cf208588e → 96a5a9702e4ae5fd after 242s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=725,stream=2616) sniff=2/0B)", "[2026-11-06T05:56:59.357Z] [poll] hard-progress: 96a5a9702e4ae5fd → 55605e2a708bcf35 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=799,stream=3023) sniff=2/0B)", "[2026-11-06T05:57:47.006Z] [poll] hard-progress: 55605e2a708bcf35 → 639dda2aa707ce04 after 48s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=4,slot=833,stream=3124) sniff=2/0B)", "[2026-11-06T05:58:16.931Z] [poll] hard-progress: 639dda2aa707ce04 → 3f35e40f26d9e4f1 after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=851,stream=3158) sniff=2/0B)", "[2026-11-06T05:58:23.051Z] [poll] hard-progress: 3f35e40f26d9e4f1 → 7a2fcc4e535e5f3f after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=858,stream=3173) sniff=2/0B)", "[2026-11-06T05:58:28.882Z] [poll] hard-progress: 7a2fcc4e535e5f3f → db60214abf828b73 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=866,stream=3191) sniff=2/0B)" ] }, "metrics": { "process": { "sampleCount": 286, "peakRssMb": 14508, "peakCpuPercent": 1205.8, "firstAtMs": 0, "lastAtMs": 1693598 }, "gpu": { "available": true, "sampleCount": 286, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 286, "peakUsedMb": 30827, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 97950, "totalOutputTokens": 11613, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 84680, "todayTokensCached": 75218, "todayTokensOut": 2086, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 84680, "totalTokensCached": 75218, "totalTokensOut": 2086, "totalCost": 0, "medianOutputTokensPerSec": 9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7824, "cachedInputTokens": 7491, "outputTokens": 122, "cost": 0, "durationMs": 92399, "outputTokensPerSec": 9.253297156255305, "at": "2026-11-06T06:20:02.758Z" }, "lastUpdated": "2026-11-06T06:20:02.758Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T06:20:06.457Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1694936, "meanTokensPerSec": 9.04, "genTokensPerSec": 9.04, "genTokens": 11613, "promptTokensPerSec": 387.29, "peakRssMb": 14508, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 84680, "output": 2086 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T05:51:45.283Z", "finishedAt": "2026-11-06T06:20:11.441Z", "timeToFirstArtifactMs": 357988, "timeToLastArtifactWriteMs": 1541038, "timeToFirstTokenMs": 35239, "firstTurnTtftMs": 21330, "timeToFirstToolCallMs": 41639 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 10, "activationEvents": 4, "completed": 8, "gateApprovals": 7, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "fieldnote-studio-office/1", "stepId": "scope", "activatedAt": "2026-11-06T05:51:52.809Z", "completedAt": "2026-11-06T05:57:43.367Z", "ms": 350558, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "9822a76d-4314-4ce5-a43c-72900cd22be9" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "draft", "activatedAt": "2026-11-06T05:57:43.368Z", "completedAt": "2026-11-06T05:57:43.711Z", "ms": 343, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "fieldnote-studio-office/1", "stepId": "collect", "activatedAt": "2026-11-06T05:57:43.712Z", "completedAt": "2026-11-06T06:13:21.282Z", "ms": 937570, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "06a05a54-5901-497d-9612-501429d8a317" ] }, { "ref": "fieldnote-studio-office/2", "stepId": "draft-invoice", "activatedAt": "2026-11-06T05:57:43.387Z", "completedAt": "2026-11-06T06:01:36.784Z", "ms": 233397, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "cea86a85-bb01-4299-89bd-493f2eef50e4" ] }, { "ref": "fieldnote-studio-office/3", "stepId": "draft-invoice", "activatedAt": "2026-11-06T05:57:43.497Z", "completedAt": "2026-11-06T06:05:39.895Z", "ms": 476398, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "6a837033-03f0-4d61-ad78-201ebe19a388" ] }, { "ref": "fieldnote-studio-office/4", "stepId": "draft-invoice", "activatedAt": "2026-11-06T05:57:43.540Z", "completedAt": "2026-11-06T06:09:39.413Z", "ms": 715873, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "4eeab0df-dc93-483a-a3eb-02adbc8f4ca7" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "evaluate", "activatedAt": "2026-11-06T06:13:21.283Z", "completedAt": "2026-11-06T06:17:26.494Z", "ms": 245211, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "9afc7f47-464f-478c-a3f3-19c82f75008d" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "finish", "activatedAt": "2026-11-06T06:17:26.495Z", "completedAt": "2026-11-06T06:20:02.701Z", "ms": 156206, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "791720fc-5d74-4a53-a9ef-e785778273ba" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T05:51:51.279Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T05:51:51.166Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 297884.5 }, "sessions": { "total": 9, "taskScoped": 7, "byGezel": { "ai": 1, "aurelien": 1, "birna": 1, "corina": 3, "mahi": 1, "sulien": 1, "taeyang": 1 }, "perTask": { "fieldnote-studio-office/1": 4, "fieldnote-studio-office/4": 1, "fieldnote-studio-office/3": 1, "fieldnote-studio-office/2": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7, "maxMessages": 2, "maxContextFill": 0.1172027587890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1172027587890625 }, "fanout": { "hosts": 1, "childrenSpawned": 3, "childrenCompleted": 3, "childrenFailed": 0, "barrierHolds": 1, "barrierReleases": 1, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-invoice-run", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "scenarioId": "craftbook-invoice-run", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-06T07:10:30.881Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/report.md", "artifactBytes": 813, "artifactSha256": "a2ff64638c96f508bb7387010572cdd178cd63287a738bd3160398c100a5bb01", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/in/invoice-run/versions/1.1.9/test.json", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ], "scoreAxes": { "numeric integrity": 8, "numbering continuity": 8, "professionalism": 5, "completeness": 4, "print-polish": 8 }, "axisJustifications": { "numeric integrity": "All three line amounts and the 5415 total match uninvoiced source entries; the prior 1200 charge is excluded. Dollar currency is assumed without a source setting.", "numbering continuity": "Generated invoices are consecutive 2026-042 through 044, but the ledger still says 041; the next run would not have a durable record of the issued sequence.", "professionalism": "Clear line items, correct terms-based due dates and readable pages. Issue dates are omitted, currency is unconfirmed, and each invoice directs questions to the customer contact instead of the issuer.", "completeness": "All three billable clients have invoices and Ondaatje is explicitly skipped. The books remain byte-identical: three entries still uninvoiced and lastInvoiceNumber unchanged.", "print-polish": "Consistent, uncluttered Letter print styles, explicit 28mm/24mm margins, aligned amounts and strong totals. All three print-CSS previews fit the printable width; PDF pagination was not asserted." }, "meanScore": 6.6, "justification": "Amounts total 5415 and invoices 042–044 are consecutive, with clear print layouts and correct due-date offsets. The ledger is unchanged, so work remains uninvoiced and the saved sequence remains 041. Issue dates are absent, currency is assumed, and invoice questions are directed to each customer rather than the issuer.", "reviewBasis": "Read the report, three invoices, roster, ledger, scope, billables, verdict, task history and write receipts. Compared the ledger against the frozen fixture and verified first two previews match final hashes; rendered and inspected the third. No model retry, ledger repair or new eval.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/report.md", "bytes": 813, "sha256": "a2ff64638c96f508bb7387010572cdd178cd63287a738bd3160398c100a5bb01", "status": "scored" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-042.html", "bytes": 4560, "sha256": "1f2adbda22af6b760e3e9a6ddfa56fc8ef0e423ae88a5b281e0a7041b42d8acc", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-043.html", "bytes": 4547, "sha256": "733142de47893d7db22af717a1c3bf1520965829851d5051cc287b8d2bb015c0", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-044.html", "bytes": 4530, "sha256": "052d544d5692fdc90d46183002c18162d1b829cf38de9c131223932af396a4bc", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/scope.md", "bytes": 1388, "sha256": "e85962d2882ec50bb2d22df2ec32d3dbdfde4c2ff9cfd756495b131b5dee6a5d", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/billables.json", "bytes": 578, "sha256": "017d6f067b37b6c7093ef0910aa775113001675565ac46ca9c3f48aeec5adab7", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/draft.md", "bytes": 248, "sha256": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205", "status": "reviewedSupportingArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/verdict.md", "bytes": 594, "sha256": "15ccee52315a88eb479a693a8f213b251d89b38ea18c6950e5039a66e6c3a74b", "status": "reviewedSupportingArtifact" } ], "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 19467, "maxSessionContextFill": 0.1172027587890625, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1172027587890625 }, "nativeFailureLines": [], "sampledPeaks": { "gpuMiB": 10640, "processTreeRssMiB": 14508, "wholeSystemRamMiB": 30827 }, "note": "Explicit CPU offload; shared GPU memory/paging unmeasured. Whole-system RAM includes other applications. Released-slot observations are not continuous KV peaks." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:24.530Z", "reviewId": "080", "cellId": "r2-saluki-cuda-complex-work-craftbook-invoice-run", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "scenarioId": "craftbook-invoice-run", "status": "scored", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-invoice-run", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "2fbedde0cb94cf9fe242ed85020b31a0dfaacb755f34b9e2c5008fbee068e0cd", "evidenceSha256": "98e17ae101227a14fcc2618a48e69a3a917cc613e93ee12b955494187a9d2265", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/report.md", "bytes": 813, "sha256": "a2ff64638c96f508bb7387010572cdd178cd63287a738bd3160398c100a5bb01", "shownBytes": 813, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-042.html", "bytes": 4560, "sha256": "1f2adbda22af6b760e3e9a6ddfa56fc8ef0e423ae88a5b281e0a7041b42d8acc", "shownBytes": 4560, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-043.html", "bytes": 4547, "sha256": "733142de47893d7db22af717a1c3bf1520965829851d5051cc287b8d2bb015c0", "shownBytes": 4547, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-044.html", "bytes": 4530, "sha256": "052d544d5692fdc90d46183002c18162d1b829cf38de9c131223932af396a4bc", "shownBytes": 4530, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/scope.md", "bytes": 1388, "sha256": "e85962d2882ec50bb2d22df2ec32d3dbdfde4c2ff9cfd756495b131b5dee6a5d", "shownBytes": 1388, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/billables.json", "bytes": 578, "sha256": "017d6f067b37b6c7093ef0910aa775113001675565ac46ca9c3f48aeec5adab7", "shownBytes": 578, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/draft.md", "bytes": 248, "sha256": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205", "shownBytes": 248, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/verdict.md", "bytes": 594, "sha256": "15ccee52315a88eb479a693a8f213b251d89b38ea18c6950e5039a66e6c3a74b", "shownBytes": 594, "truncated": false } ], "contextFiles": [], "scoreAxes": { "numeric integrity": 8, "numbering continuity": 9, "professionalism": 7, "completeness": 6, "print-polish": 8 }, "justification": "Amounts ($1,840, $2,600, $975; total $5,415) match scope.md and billables.json. Numbers 042–044 follow 041. Due dates are right for net 30/14/30 from 2026-11-05. Each invoice has an @page rule, margins and Helvetica. Gaps: no issue date, one line item each, and an odd \"[6]\" citation in the report.", "limitations": "The ledger.json and clients.json sources weren't shown, so amounts were checked only against scope.md. No evidence the books were updated, such as a last-number bump or ledger entries marked invoiced. No rendering was done.", "meanScore": 7.6, "durationMs": 17600, "costUsd": 0.0529332, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 11690, "cache_read_input_tokens": 892, "output_tokens": 608, "output_tokens_details": { "thinking_tokens": 249 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 11690, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 608, "cache_read_input_tokens": 892, "cache_creation_input_tokens": 11690, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 11690 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 608, "cacheReadInputTokens": 892, "cacheCreationInputTokens": 11690, "webSearchRequests": 0, "costUSD": 0.0529332, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 249, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 11690, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-invoice-run / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 28m 26s; budget used: 22.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 357988`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":809,\"score\":10,\"scoreMax\":10,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2272033656009472`; `facts.toolUse.totalToolCalls: 41`; `facts.timing.timeToFirstArtifactMs: 357988`; `facts.outcome.durationMs: 1706158` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14508 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30827 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.04 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 97950 + 11613 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 10 → 8 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 298s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 7 / 0.70 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 12% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 3 / 3 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 1 / 1 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 4 of 6 / 20 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 357988`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":809,\"score\":10,\"scoreMax\":10,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.2272033656009472`\n- `facts.toolUse.totalToolCalls: 41`\n- `facts.timing.timeToFirstArtifactMs: 357988`\n- `facts.outcome.durationMs: 1706158`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 357988`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":809,\"score\":10,\"scoreMax\":10,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2272033656009472`; `facts.toolUse.totalToolCalls: 41`; `facts.timing.timeToFirstArtifactMs: 357988`; `facts.outcome.durationMs: 1706158`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.5/10**, all 10 checks, remains unchanged. Advisory **6.6/10**, codex-inline/current-session.\n\n| Axis | Score | Evidence |\n|---|---:|---|\n| numeric integrity | 8 | All three line amounts and the 5415 total match uninvoiced source entries; the prior 1200 charge is excluded. Dollar currency is assumed without a source setting. |\n| numbering continuity | 8 | Generated invoices are consecutive 2026-042 through 044, but the ledger still says 041; the next run would not have a durable record of the issued sequence. |\n| professionalism | 5 | Clear line items, correct terms-based due dates and readable pages. Issue dates are omitted, currency is unconfirmed, and each invoice directs questions to the customer contact instead of the issuer. |\n| completeness | 4 | All three billable clients have invoices and Ondaatje is explicitly skipped. The books remain byte-identical: three entries still uninvoiced and lastInvoiceNumber unchanged. |\n| print-polish | 8 | Consistent, uncluttered Letter print styles, explicit 28mm/24mm margins, aligned amounts and strong totals. All three print-CSS previews fit the printable width; PDF pagination was not asserted. |\n\nAmounts total 5415 and invoices 042–044 are consecutive, with clear print layouts and correct due-date offsets. The ledger is unchanged, so work remains uninvoiced and the saved sequence remains 041. Issue dates are absent, currency is assumed, and invoice questions are directed to each customer rather than the issuer.\n\nRead the report, three invoices, roster, ledger, scope, billables, verdict, task history and write receipts. Compared the ledger against the frozen fixture and verified first two previews match final hashes; rendered and inspected the third. No model retry, ledger repair or new eval.\n\n## Positives\n\n- All 10 deterministic checks pass. The host completes five stages and all three child invoice tasks complete, with a held/released fanout barrier and no failed child (outcome.success; continuity.steps.completed=8; continuity.fanout.childrenCompleted=3).\n- Each previously uninvoiced entry becomes one invoice: 1840, 2600 and 975, total 5415. The previously invoiced 1200 entry is not charged again; the idle client is explicitly skipped (artifacts.htmlFiles; report.md; fixture/amount audit).\n- Numbers 2026-042 through 044 are consecutive, and net 30/net 14 due dates agree with the local issue-date basis documented in scope. The three paper layouts are readable and consistent with explicit page margins (artifacts.htmlFiles; reviewed CSS/screenshots).\n- The actual workflow reaches terminal completion after review and a DONE note. Unlike R1, the verdict has no extraneous footer after its criterion lines, and no workflow gate rejection occurs (continuity.steps.gateRejections=0; verdict.md; recording/task-notes.json).\n\n## Negatives\n\n- The ledger is byte-identical to the seed: lastInvoiceNumber remains 2026-041 and all three billable entries remain invoiced:false. No ledger write occurs. The requested bookkeeping objective remains incomplete despite the fixed pass (toolUse.byTool.write_file=5; captured ledger; advisory-consistency-audit.json).\n- The catalog workflow omits a books-update action in collect/evaluate, and test.json checks only that ledger.json parses. This is a content/evaluator coverage gap; the evidence does not isolate a model refusal to perform an explicit ledger-update step (sniff.latest.score=10; captured craftbook prompts and test.json).\n- Every invoice routes Questions about this invoice to its customer billing address, not an issuer contact. None prints an explicit issue date, and all assume dollars despite no currency in the inputs (artifacts.htmlFiles; source/visual audit).\n- The report says invoices went out, but only local artifact creation is evidenced. No delivery channel was configured or used; distinguish generated invoices from sent invoices (toolUse.byTool; report.md).\n- A report write is rejected for the already-supported total $5,415.00, with the guard asking for evidence of the fragment 415.00; reading scope and adding a citation resolves it. This is an evidence-format/numeric-tokenization issue, not incorrect arithmetic (advisory-tool-evidence.json; numericGuardRejection in audit).\n\n## Performance and GPU fit\n\n**28m26s**, **41 calls**, **11,613 engine-generated tokens at 9.04 tok/s**, prefill **387.29 tok/s** (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10,640/12,227 MiB**, process-tree RSS **14,508 MiB**, whole-system RAM **30,827/31,865 MiB**. Whole-system RAM includes other applications; shared GPU memory and paging were not measured. No new campaign memory peak.\n\nOne native 0.1.57 CUDA launch, 65,536 context, one slot and f16 KV. Dense FFN placement remains 39/64 GPU and 25 CPU: hybrid residency. Session context fill peaks at 11.7%; largest observed released slot **19467 tokens (29.7%)**. No compaction, endpoint failure, explicit OOM or context overflow observed. Native slot observations are not continuous KV peaks.\n\n## Tactical experiments — not applied\n\nIn ../gilde/data/craftbook-templates/in/invoice-run/versions/1.1.9/craftbook.json, collect.prompt should explicitly require recording successful local invoice generation in the books, with the invoice number on each affected entry and the new lastInvoiceNumber, while preserving the previously invoiced entry. evaluate.prompt must verify those persisted changes independently. Add to scope/draft prompts: “Record the issue date and currency basis. Use issuer contact details only for invoice questions; if not supplied, omit that line. Describe local artifacts as generated, not sent, without a delivery receipt.” Evidence: unchanged ledger, footer confusion and unconfirmed currency in the captured outputs. Version such changes separately; none were applied here.\n\n## Strategic experiments — not applied\n\nAdd a deterministic bookkeeping commit operation under packages/script-stdlib/scripts/ with an idempotency key tied to source entries and assigned invoice numbers, invoked only after invoice validation. Its acceptance check should compare original/final ledger states and reject duplicate numbering or a second charge of previously invoiced work. In the catalog test.json success.checks, add semantic reconciliation rather than only json-valid. These proposed checks must use arbitrary fixtures and not encode these client names/numbers.\n\nIn packages/core/src/checks/claims.ts, test thousands-separated currency tokens as a single numeric claim and preserve the cited derivation of totals. Do not suppress genuinely unsupported claims.\n\n## Generalization and next experiment\n\nR1 failed fixed 5.5 despite correct invoice files because the verdict's prose footer violated its exact line format; advisory was 6.8 (8/8/6/4/8), and the books were also unchanged. R2 passes fixed 8.5 with a clean verdict but still leaves the bookkeeping objective incomplete. The advisory 6.6 reflects the additional explicit misdirected-contact issue; it is not evidence of a native-version change.\n\nUnrelated TDD/refactor show that completion prose can diverge from actual edits, while A/B reporting showed a correct structured answer alongside an inconsistent report. Evaluate receipt/state-bound completion checks across invoices, record cleanup and one development workflow. Measure state consistency, duplicate-write safety, false rejections and tool overhead without changing this frozen campaign.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/facts.json): fail, fixed 5.5, 58.74 min, 8.44 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/facts.json): pass, fixed 8.5, 3.21 min, 142.73 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-01-17-689Z-7dqo/facts.json): pass, fixed 9, 3.29 min, 137.89 tok/s.\n\nOther-model history is context, not a controlled native-version comparison.\n\n## Strategic note\n\nThe model and runtime successfully generated three accurate invoices through real fanout, review and completion. Durable bookkeeping remains unimplemented by the current workflow/checks. Do not equate the fixed pass with a complete production invoicing run. No source, tuning, gate, prompt or ledger edits, model retries or additional evals were performed by this review.\n\n## Raw evidence\n\nfacts.json, score.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json, recorded task state, captured artifacts/workspace and the two advisory preview folders.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-author-linear", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-author-linear", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T06:22:58.630Z", "finishedAt": "2026-11-06T07:29:57.954Z", "durationMs": 4019324, "success": true, "reason": "authored a gated 3-step craftbook, invoked it, ran the task to completion, and all three outputs pass their checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1396172 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-linear", "bytes": 7758, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.33452498077614357 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 63 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1396172 }, { "source": "facts", "path": "outcome.durationMs", "value": 4019324 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 675, "peakRssMb": 16554, "peakCpuPercent": 2983436.6, "firstAtMs": 0, "lastAtMs": 4004082 }, "gpu": { "available": true, "sampleCount": 675, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 675, "peakUsedMb": 31245, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 152749, "totalOutputTokens": 27469, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 171681, "todayTokensCached": 168290, "todayTokensOut": 2946, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 171681, "totalTokensCached": 168290, "totalTokensOut": 2946, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 16691, "cachedInputTokens": 16362, "outputTokens": 64, "cost": 0, "durationMs": 1819569, "outputTokensPerSec": 8.47864679720463, "at": "2026-11-06T07:29:44.879Z" }, "lastUpdated": "2026-11-06T07:29:44.879Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T07:29:50.575Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4005751, "meanTokensPerSec": 8.25, "genTokensPerSec": 8.25, "genTokens": 27469, "promptTokensPerSec": 331.5, "peakRssMb": 16554, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 171681, "output": 2946 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "order-intake-cleanup/1", "stepId": "inspect", "activatedAt": "2026-11-06T06:38:04.174Z", "completedAt": "2026-11-06T06:48:51.043Z", "ms": 646869, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "09a24384-266b-4869-a749-d6966014900b" ] }, { "ref": "order-intake-cleanup/1", "stepId": "clean", "activatedAt": "2026-11-06T06:48:51.045Z", "completedAt": "2026-11-06T06:58:21.702Z", "ms": 570657, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "9dbdcd83-b66a-47d3-9051-56b033c9c77f" ] }, { "ref": "order-intake-cleanup/1", "stepId": "verify", "activatedAt": "2026-11-06T06:58:21.703Z", "completedAt": "2026-11-06T07:29:44.785Z", "ms": 1883082, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "958f69a5-1938-435e-ad37-3d3ecbe5dafa" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T06:23:04.463Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T06:23:04.361Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 646869 }, "sessions": { "total": 5, "taskScoped": 3, "byGezel": { "arman": 1, "aurel": 2, "mindaugas": 1, "reza": 1 }, "perTask": { "order-intake-cleanup/1": 3 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6, "maxMessages": 11, "maxContextFill": 0.410797119140625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.410797119140625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/reorder-run", "knowledge://bendyline/handboek/whats-new/1.26273", "knowledge://bendyline/handboek/craftbook/scrape-to-structured" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "authored a gated 3-step craftbook, invoked it, ran the task to completion, and all three outputs pass their checks", "durationMs": 4019324, "timeoutMs": 12015019, "budgetUsedFraction": 0.33452498077614357 }, "timing": { "startedAt": "2026-11-06T06:22:58.630Z", "finishedAt": "2026-11-06T07:29:57.954Z", "timeToFirstArtifactMs": 1396172, "timeToLastArtifactWriteMs": 4006250, "timeToFirstTokenMs": 47306, "firstTurnTtftMs": 36898, "timeToFirstToolCallMs": 97024 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Data-Analyst", "Developer", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 63, "byTool": { "list_craftbooks": 2, "craftbook_read": 2, "craftbook_write": 2, "invoke_craftbook": 1, "list_tasks": 2, "read_task_notes": 8, "stat": 13, "search_history": 1, "get_task": 5, "read_file": 13, "list_dir": 1, "list_project_gezels": 1, "delegate_reviewer": 1, "write_file": 5, "write_task_note": 3, "advance_task_step": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 7679, "filePath": "craftbook-author-linear", "bytes": 0, "score": 3, "scoreMax": 7, "signals": [], "failReason": "no reusable authored craftbook with at least 3 steps exists yet — author \"CSV Order Cleanup\" (inspect, clean, verify) as a craftbook template, not as ad-hoc chat work" }, { "atMs": 872284, "filePath": "craftbook-author-linear", "bytes": 28, "score": 3, "scoreMax": 7, "signals": [], "failReason": "no task has been created from craftbook \"csv-order-cleanup\" yet — invoke the craftbook on this project" }, { "atMs": 907871, "filePath": "craftbook-author-linear", "bytes": 28, "score": 3, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 1400629, "filePath": "craftbook-author-linear", "bytes": 1822, "score": 4, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 1992346, "filePath": "craftbook-author-linear", "bytes": 5263, "score": 5, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3816087, "filePath": "craftbook-author-linear", "bytes": 7758, "score": 6, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 4011493, "filePath": "craftbook-author-linear", "bytes": 7758, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-author-linear", "bytes": 7758, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 4011483, "kind": "structured", "gezel": "reza", "question": "**CSV Order Cleanup — data/orders.csv** is finished. - **`out/report.md`** - `o…", "chose": "write-in (workspace-fixture)" } ] }, "miscEvents": [ "[2026-11-06T06:22:58.634Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "[2026-11-06T06:22:58.634Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-linear-2DB2Ip", "[2026-11-06T06:22:58.634Z] [trial] id=craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz scenario=craftbook-author-linear model=underdog-saluki-27b-iq2-mix", "[2026-11-06T06:22:58.634Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T06:22:58.634Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T06:22:58.636Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T06:22:58.640Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T06:22:58.647Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-linear-2DB2Ip", "[2026-11-06T06:22:58.647Z] [trial] throughput-scaled ceiling: 80m → 200m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T06:23:04.460Z] [trial] daemon spawned pid=32428 port=https://127.0.0.1:57718", "[2026-11-06T06:23:05.714Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T06:23:05.731Z] [trial] meester=florian", "[2026-11-06T06:23:05.731Z] [trial] running scenario setup hook", "[2026-11-06T06:23:06.079Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T06:23:06.077Z] [trial] scenario setup complete", "[2026-11-06T06:23:06.142Z] [poll] initial digests hard=9deb8c4ae1c2ec4c soft=5576b2c11877d957", "[2026-11-06T06:23:06.440Z] [poll] hard-progress: 9deb8c4ae1c2ec4c → 39e23a8f8b2e926c after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=3/0B)", "[2026-11-06T06:24:40.225Z] [poll] hard-progress: 39e23a8f8b2e926c → 816a4ac0af48158e after 94s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=162,stream=431) sniff=3/0B)", "[2026-11-06T06:24:58.023Z] [poll] hard-progress: 816a4ac0af48158e → 1b58212d68aeeeb8 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=184,stream=490) sniff=3/0B)", "[2026-11-06T06:33:03.332Z] [poll] hard-progress: 1b58212d68aeeeb8 → 6bd169a34b4a6705 after 485s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=1449,stream=5481) sniff=3/0B)", "[2026-11-06T06:37:31.026Z] [poll] hard-progress: 6bd169a34b4a6705 → f02f546d2d208557 after 268s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=2139,stream=9106) sniff=3/28B)", "[2026-11-06T06:38:06.565Z] [poll] hard-progress: f02f546d2d208557 → 49bdf91c8d86661b after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=2198,stream=9389) sniff=3/28B)", "[2026-11-06T06:38:24.398Z] [poll] hard-progress: 49bdf91c8d86661b → 79e09c96536333b5 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=2237,stream=9504) sniff=3/28B)", "[2026-11-06T06:38:36.214Z] [poll] hard-progress: 79e09c96536333b5 → 264f0befe7a63f5e after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=2261,stream=9576) sniff=3/28B)", "[2026-11-06T06:38:48.162Z] [poll] hard-progress: 264f0befe7a63f5e → 5c21aa0f2c69e0b3 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=2284,stream=9649) sniff=3/28B)" ] }, "metrics": { "process": { "sampleCount": 675, "peakRssMb": 16554, "peakCpuPercent": 2983436.6, "firstAtMs": 0, "lastAtMs": 4004082 }, "gpu": { "available": true, "sampleCount": 675, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 675, "peakUsedMb": 31245, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 152749, "totalOutputTokens": 27469, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 7, "todayTokensIn": 171681, "todayTokensCached": 168290, "todayTokensOut": 2946, "todayCost": 0, "totalTurns": 7, "totalTokensIn": 171681, "totalTokensCached": 168290, "totalTokensOut": 2946, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 7 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 16691, "cachedInputTokens": 16362, "outputTokens": 64, "cost": 0, "durationMs": 1819569, "outputTokensPerSec": 8.47864679720463, "at": "2026-11-06T07:29:44.879Z" }, "lastUpdated": "2026-11-06T07:29:44.879Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T07:29:50.575Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4005751, "meanTokensPerSec": 8.25, "genTokensPerSec": 8.25, "genTokens": 27469, "promptTokensPerSec": 331.5, "peakRssMb": 16554, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 171681, "output": 2946 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T06:22:58.630Z", "finishedAt": "2026-11-06T07:29:57.954Z", "timeToFirstArtifactMs": 1396172, "timeToLastArtifactWriteMs": 4006250, "timeToFirstTokenMs": 47306, "firstTurnTtftMs": 36898, "timeToFirstToolCallMs": 97024 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "order-intake-cleanup/1", "stepId": "inspect", "activatedAt": "2026-11-06T06:38:04.174Z", "completedAt": "2026-11-06T06:48:51.043Z", "ms": 646869, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "09a24384-266b-4869-a749-d6966014900b" ] }, { "ref": "order-intake-cleanup/1", "stepId": "clean", "activatedAt": "2026-11-06T06:48:51.045Z", "completedAt": "2026-11-06T06:58:21.702Z", "ms": 570657, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "9dbdcd83-b66a-47d3-9051-56b033c9c77f" ] }, { "ref": "order-intake-cleanup/1", "stepId": "verify", "activatedAt": "2026-11-06T06:58:21.703Z", "completedAt": "2026-11-06T07:29:44.785Z", "ms": 1883082, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "958f69a5-1938-435e-ad37-3d3ecbe5dafa" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T06:23:04.463Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T06:23:04.361Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 646869 }, "sessions": { "total": 5, "taskScoped": 3, "byGezel": { "arman": 1, "aurel": 2, "mindaugas": 1, "reza": 1 }, "perTask": { "order-intake-cleanup/1": 3 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6, "maxMessages": 11, "maxContextFill": 0.410797119140625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.410797119140625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-author-linear", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "scenarioId": "craftbook-author-linear", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-06T08:06:45.394Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/out/report.md", "artifactBytes": 2531, "artifactSha256": "9c74025ac2a91380e3402f4e2ebfcc4ecaf17b35028b94f747a43cfd3998aa15", "axisSource": "evals/src/craftbooks/authoring/author-linear.ts", "axes": [], "scoreAxes": {}, "axisJustifications": {}, "meanScore": null, "justification": "No declared axes. All 28 valid records retain every field correctly; the two malformed rows are documented and excluded. The named three-step recipe completes, and the final report reconciles counts and five accurate comparisons. Its gates remain size/nonempty-only, so the promised FAIL-to-clean routing is not enforced.", "reviewBasis": "Read all three final outputs, the CSV fixture, accepted recipe and final task state. Compared every retained field to its source, verified the report examples, inspected write/advance receipts, and checked compaction evidence. No model retry or negative-path execution.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/notes/anomalies.md", "bytes": 1812, "sha256": "5e8553e63a4ee5e618f483491f4ea9e585cdbeded3b336271c4b88ec24ed8861", "status": "reviewedNoDeclaredAxes" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/out/orders.json", "bytes": 3441, "sha256": "48b428197919ee82d35eacc53deebe9545fb28e4e749d23184fabefea409d788", "status": "reviewedNoDeclaredAxes" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/out/report.md", "bytes": 2531, "sha256": "9c74025ac2a91380e3402f4e2ebfcc4ecaf17b35028b94f747a43cfd3998aa15", "status": "reviewedNoDeclaredAxes" } ], "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 42445, "maxSessionContextFill": 0.410797119140625, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.410797119140625 }, "compactionEvidence": [ "2026-11-06T06:56:31.261Z INFO [chat] [chat] deterministic mid-loop compaction for session 34895e3b (35555/65536 estimated tokens)" ], "nativeFailureLines": [], "sampledPeaks": { "gpuMiB": 10644, "processTreeRssMiB": 16554, "wholeSystemRamMiB": 31245 }, "note": "One successful deterministic mid-loop compaction, not context overflow. CPU offload explicit; shared GPU memory/paging unmeasured. Whole-system RAM includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:24.448Z", "reviewId": "081", "cellId": "r2-saluki-cuda-complex-work-craftbook-author-linear", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "scenarioId": "craftbook-author-linear", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-author-linear", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "c36ed1b84d7dd2e8a43e5f995c15273f64b4bd67f25ea8579abb2ca7a1fdd306", "evidenceSha256": "03d00e403eb3943a6c8f2d4e2bd602bb076159c5ba08a7a1de81a8a8c663fa8d", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/out/report.md", "bytes": 2531, "sha256": "9c74025ac2a91380e3402f4e2ebfcc4ecaf17b35028b94f747a43cfd3998aa15", "shownBytes": 2531, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/notes/anomalies.md", "bytes": 1812, "sha256": "5e8553e63a4ee5e618f483491f4ea9e585cdbeded3b336271c4b88ec24ed8861", "shownBytes": 1812, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/out/orders.json", "bytes": 3441, "sha256": "48b428197919ee82d35eacc53deebe9545fb28e4e749d23184fabefea409d788", "shownBytes": 3441, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz\\workspace\\order-intake-cleanup\\data\\orders.csv", "bytes": 1546, "sha256": "c88785ed78c4d8b14484d8d0f04516c9c93c99045b2e90b60b19f96ee17d7bb2", "shownBytes": 1546, "truncated": false } ], "scoreAxes": {}, "justification": "Outputs are correct against the CSV: the two malformed rows (ORD-1013 qty \"two\", ORD-1022 date 2026-13-45) were found and dropped, and 28 valid records were kept (≥20). The anomalies.md, orders.json and report.md are consistent with each other.", "limitations": "The evidence does not show the craftbook document, its step deliverable declarations, the craftbook_write call, the invoke_craftbook call or task completion. Authoring-before-delegation and gating cannot be verified. Only the output files were reviewed.", "meanScore": null, "durationMs": 14964, "costUsd": 0.035196, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 8033, "cache_read_input_tokens": 700, "output_tokens": 299, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 8033, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 299, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 8033, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 8033 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 299, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 8033, "webSearchRequests": 0, "costUSD": 0.035196, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 8033, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-linear / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 66m 59s; budget used: 33.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1396172`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7758,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.33452498077614357`; `facts.toolUse.totalToolCalls: 63`; `facts.timing.timeToFirstArtifactMs: 1396172`; `facts.outcome.durationMs: 4019324` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 16554 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31245 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.25 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 152749 + 27469 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 647s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 3 / 0.60 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 1 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 41% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1396172`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7758,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.33452498077614357`\n- `facts.toolUse.totalToolCalls: 63`\n- `facts.timing.timeToFirstArtifactMs: 1396172`\n- `facts.outcome.durationMs: 4019324`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1396172`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7758,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.33452498077614357`; `facts.toolUse.totalToolCalls: 63`; `facts.timing.timeToFirstArtifactMs: 1396172`; `facts.outcome.durationMs: 4019324`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.5/10**, all 7 checks, unchanged. No declared qualitative axes; advisory **notApplicable**, with all three produced outputs reviewed.\n\nNo declared axes. All 28 valid records retain every field correctly; the two malformed rows are documented and excluded. The named three-step recipe completes, and the final report reconciles counts and five accurate comparisons. Its gates remain size/nonempty-only, so the promised FAIL-to-clean routing is not enforced.\n\nRead all three final outputs, the CSV fixture, accepted recipe and final task state. Compared every retained field to its source, verified the report examples, inspected write/advance receipts, and checked compaction evidence. No model retry or negative-path execution.\n\n## Positives\n\n- A reusable template named CSV Order Cleanup is created and invoked in the seeded project. All three task stages complete with their declared deliverables and no gate rejection (outcome.success; sniff.latest.score=7; continuity.steps.completed=3; final task state).\n- Every field in all 28 retained records agrees with the source, with numeric quantities/prices and unchanged identifiers, names and valid dates. Only ORD-1013 and ORD-1022 are omitted, matching the documented drop policy (artifacts.otherFileCount; advisory-consistency-audit.json).\n- The final report accurately reconciles 30 minus 2 equals 28, names the correct source line numbers/reasons and gives five correct field comparisons. The earlier anomaly report has no wrong customer name (captured report and anomaly log; prior-checkpoint hash matches).\n- The inputPath parameter is actually interpolated in the inspect prompt and the requested recipe name is preserved, improving reuse relative to R1. A deterministic context compaction succeeds without a runtime failure (authored recipe; final task prompt; continuity.compaction.midTurn=1).\n\n## Negatives\n\n- The clean gate only checks size/nonempty, not JSON semantics. Verify likewise accepts any sufficiently long nonempty report, even if it says FAIL; terminal:true then closes the task. The prompt promises a retry that the gate cannot enforce. This is an unexercised structural weakness, not a failure of the observed correct output (finalTask.craftbook.steps; audit.verificationRoutingGap).\n- The first Markdown submission fails because the configured writer expects JSON; a full JSON resubmission succeeds about 4m29s later. The author had explicitly requested a Markdown read, so reading and writing used different formats (toolUse.byTool.craftbook_write=2; recorded receipts).\n- Two otherwise grounded writes are rejected by the evidence guard: derived row counts/line numbers and ordinary IDs/SKUs labels, then the task minimum 20 and the 8601 in ISO-8601. Added citations resolve both. These are write guards, not workflow gate rejections (advisory-tool-evidence.json; continuity.steps.gateRejections=0).\n- Sixty-three calls and 66m59s include 13 stats, repeated task/note queries and a second reviewer handoff while the task reviewer is already working. Verify takes about 31m23s; the report repair alone spans roughly 11m40s (toolUse.byTool; continuity.steps.perStep; write chronology).\n\n## Performance and GPU fit\n\n**66m59s**, **63 calls**, **27,469 engine-generated tokens at 8.25 tok/s**, prefill **331.5 tok/s** (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Peak GPU **10,644/12,227 MiB**, process-tree RSS **16,554 MiB**, whole-system RAM **31,245/31,865 MiB**. Whole-system RAM includes other applications; shared GPU memory/paging unmeasured. No new campaign peak.\n\nOne native 0.1.57 CUDA launch, 65,536 context, one slot and f16 KV. Dense FFN placement: 39/64 layers on GPU and 25 on CPU. Maximum recorded session fill 41.1%; largest released native slot **42445 tokens (64.8%)**. The provider logs one successful deterministic mid-loop compaction at an estimated 35,555/65,536 tokens at 06:56:31. This is normal context management, not overflow. No native endpoint failure, explicit OOM or context overflow observed.\n\n## Tactical experiments — not applied\n\nIn evals/src/craftbooks/authoring/helpers.ts AUTHORING_TOOL_STEER, add a generic instruction to follow the writer's configured serialization contract, rather than assuming an explicitly requested read format is also accepted by write. Measure invalid-document calls across multiple authoring tasks before retaining this nudge.\n\nFor future authored recipes, use a JSON-valid/schema/count check on the clean deliverable and a machine-readable PASS/FAIL verification result with explicit rejection routing. In the captured csv-order-cleanup verify.gate.checks, a future separate experiment would require a PASS verdict and reject FAIL; currently it only tests minBytes/nonempty. This proposes a structural recipe capability, not a repair applied to this completed trial.\n\n## Strategic experiments — not applied\n\nIn packages/core/src/checks/claims.ts, distinguish derived counts and terminology (IDs, SKUs, ISO-8601) from unsupported measured facts, and include task acceptance criteria as evidence for stated thresholds. Keep genuine factual grounding strict. The invoice trial's 415.00 fragment rejection gives an unrelated numeric-token case to test alongside this report.\n\nIn packages/service/src/chat/manager.ts, expose whether a task's assigned reviewer is already queued/running before allowing a parallel handoff for the same task and deliverable. Preserve useful independent review while measuring duplicate turns and latency.\n\n## Generalization and next experiment\n\nR1 also passed fixed 8.5 with all 28 valid rows, but its anomaly log named the wrong customer, the final report corrected that error, and its declared input parameter was unused. R2's documents agree, the exact requested recipe name is used and inputPath is interpolated. Both retain size/nonempty-only gates. These differences are repetition variability under unchanged native/tuning identity.\n\nUnrelated invoice-run exposes a semantic gate gap despite good artifacts, and TDD/refactor show why completed steps need receipt-backed evidence. Test generic structured validation, explicit negative-verdict routing and redundant-review suppression across data, invoicing and development workflows. Measure false passes, false rejections, source fidelity and latency; do not hardcode this dataset or its anomalous IDs.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/facts.json): pass, fixed 8.5, 41.76 min, 8.31 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/facts.json): fail, fixed 5.4, 17.33 min, 112.19 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-11-17-137Z-am5s/facts.json): pass, fixed 8.5, 6.49 min, 113.38 tok/s.\n\nOther-model history is contextual, not a controlled native-version comparison.\n\n## Strategic note\n\nThis is real recipe authorship and execution with faithful data transformation, beyond model loading or task routing. The remaining weaknesses are untested negative-path enforcement and expensive coordination/evidence repair. No source, prompts, gates, tuning, model retries or additional evals were applied by this review.\n\n## Raw evidence\n\nfacts.json, score.json, state.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json and captured workspace.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-route-multi", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-route-multi", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T07:33:08.380Z", "finishedAt": "2026-11-06T08:07:38.970Z", "durationMs": 2070590, "success": true, "reason": "invoked the bundled press-release recipe, ran the task to completion, and the release carries a dateline, the attributed approved quote, the boilerplate, a contact, and the approved facts rather than the stale draft", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 780125 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-route-multi", "bytes": 995, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.17233347695912923 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 33 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 780125 }, { "source": "facts", "path": "outcome.durationMs", "value": 2070590 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 347, "peakRssMb": 14756, "peakCpuPercent": 1228.6, "firstAtMs": 0, "lastAtMs": 2059392 }, "gpu": { "available": true, "sampleCount": 347, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 347, "peakUsedMb": 31277, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 84510, "totalOutputTokens": 15113, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 104668, "todayTokensCached": 98733, "todayTokensOut": 7320, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 104668, "totalTokensCached": 98733, "totalTokensOut": 7320, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 9 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3480, "outputTokens": 293, "cost": 0, "durationMs": 67822, "outputTokensPerSec": 9.090741823523732, "at": "2026-11-06T08:07:33.307Z" }, "lastUpdated": "2026-11-06T08:07:33.307Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T08:07:35.397Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2060690, "meanTokensPerSec": 8.75, "genTokensPerSec": 8.75, "genTokens": 15113, "promptTokensPerSec": 390.52, "peakRssMb": 14756, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 104668, "output": 7320 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "fieldstone-launch-comms/1", "stepId": "scope", "activatedAt": "2026-11-06T07:34:37.670Z", "completedAt": "2026-11-06T07:46:08.665Z", "ms": 690995, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "d3ed61a1-c454-40f6-85bd-cf9a0a79d32f" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "draft", "activatedAt": "2026-11-06T07:46:08.666Z", "completedAt": "2026-11-06T07:48:31.815Z", "ms": 143149, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "24dd97fc-b9f9-4a3a-824b-4ff5bcf1ae27" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "review", "activatedAt": "2026-11-06T07:48:31.816Z", "completedAt": "2026-11-06T07:53:23.538Z", "ms": 291722, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "cc99ffba-14d1-4d71-8e04-231ec0ecd5b5" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "evaluate", "activatedAt": "2026-11-06T07:53:23.539Z", "completedAt": "2026-11-06T08:02:07.273Z", "ms": 523734, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "cc99ffba-14d1-4d71-8e04-231ec0ecd5b5" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "finish", "activatedAt": "2026-11-06T08:02:07.275Z", "completedAt": "2026-11-06T08:06:20.034Z", "ms": 252759, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "f98b558a-06d2-4c10-a8fe-ad99736ce136" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T07:33:14.278Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T07:33:14.165Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 291722 }, "sessions": { "total": 6, "taskScoped": 4, "byGezel": { "ines": 1, "juan": 1, "mykola": 1, "shahd": 1, "zain": 2 }, "perTask": { "fieldstone-launch-comms/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 9, "maxContextFill": 0.2747344970703125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2747344970703125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/gezel-in-docker", "knowledge://bendyline/handboek/notifications", "knowledge://bendyline/handboek/adding-folders", "knowledge://bendyline/handboek/craftbook/chapter-weave" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "invoked the bundled press-release recipe, ran the task to completion, and the release carries a dateline, the attributed approved quote, the boilerplate, a contact, and the approved facts rather than the stale draft", "durationMs": 2070590, "timeoutMs": 12015019, "budgetUsedFraction": 0.17233347695912923 }, "timing": { "startedAt": "2026-11-06T07:33:08.380Z", "finishedAt": "2026-11-06T08:07:38.970Z", "timeToFirstArtifactMs": 780125, "timeToLastArtifactWriteMs": 2064927, "timeToFirstTokenMs": 48227, "firstTurnTtftMs": 27701, "timeToFirstToolCallMs": 89452 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 33, "byTool": { "start_work": 1, "get_task": 1, "read_task_notes": 2, "list_artifacts": 2, "read_artifact": 5, "read_file": 8, "list_dir": 3, "write_artifact": 2, "write_file": 2, "advance_task_step": 3, "read_files": 1, "replace_lines": 1, "validate": 1, "write_task_note": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 7711, "filePath": "craftbook-route-multi", "bytes": 0, "score": 9, "scoreMax": 11, "signals": [], "failReason": "no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" }, { "atMs": 90048, "filePath": "craftbook-route-multi", "bytes": 0, "score": 9, "scoreMax": 11, "signals": [], "failReason": "task fieldstone-launch-comms/1 (from \"press-release\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 1994981, "filePath": "craftbook-route-multi", "bytes": 0, "score": 10, "scoreMax": 11, "signals": [], "failReason": "out/press-release.md is 0 bytes, need ≥ 600" }, { "atMs": 2066526, "filePath": "craftbook-route-multi", "bytes": 995, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-route-multi", "bytes": 995, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 1994972, "kind": "structured", "gezel": "ines", "question": "Your document is finished. - **`press-release.md`** - `tasks/1/verdict.md` - `t…", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-06T07:33:08.384Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-route-multi-USXiDD", "[2026-11-06T07:33:08.384Z] [trial] id=craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t scenario=craftbook-route-multi model=underdog-saluki-27b-iq2-mix", "[2026-11-06T07:33:08.384Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "[2026-11-06T07:33:08.384Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T07:33:08.384Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T07:33:08.387Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T07:33:08.391Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T07:33:08.397Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-route-multi-USXiDD", "[2026-11-06T07:33:08.397Z] [trial] throughput-scaled ceiling: 80m → 200m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T07:33:14.266Z] [trial] daemon spawned pid=53712 port=https://127.0.0.1:61257", "[2026-11-06T07:33:15.503Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T07:33:15.519Z] [trial] running scenario setup hook", "[2026-11-06T07:33:15.519Z] [trial] meester=ines", "[2026-11-06T07:33:15.849Z] [trial] scenario setup complete", "[2026-11-06T07:33:15.851Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T07:33:15.920Z] [poll] initial digests hard=6fd874e505ac522c soft=e02b831da836d59a", "[2026-11-06T07:33:16.172Z] [poll] hard-progress: 6fd874e505ac522c → 43da4187a289c3c6 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-06T07:34:38.517Z] [poll] hard-progress: 43da4187a289c3c6 → 595e3c00eb286a55 after 82s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=126,stream=438) sniff=9/0B)", "[2026-11-06T07:35:07.785Z] [poll] hard-progress: 595e3c00eb286a55 → 13744295d130b368 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=196,stream=644) sniff=9/0B)", "[2026-11-06T07:35:43.320Z] [poll] hard-progress: 13744295d130b368 → 822fa6764e58c9c3 after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=3,slot=231,stream=734) sniff=9/0B)", "[2026-11-06T07:35:49.579Z] [poll] hard-progress: 822fa6764e58c9c3 → 285bc9d3b952edd4 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=234,stream=735) sniff=9/0B)", "[2026-11-06T07:36:19.342Z] [poll] hard-progress: 285bc9d3b952edd4 → 30f58970cd4e897a after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=256,stream=796) sniff=9/0B)", "[2026-11-06T07:39:45.764Z] [poll] hard-progress: 30f58970cd4e897a → eca8c9097733ca1a after 206s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=831,stream=2507) sniff=9/0B)", "[2026-11-06T07:40:45.524Z] [poll] hard-progress: eca8c9097733ca1a → 973025b7d3f07c87 after 60s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=902,stream=2713) sniff=9/0B)", "[2026-11-06T07:46:13.597Z] [poll] hard-progress: 973025b7d3f07c87 → 7338de4cafe3f85f after 328s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=10,slot=1827,stream=6175) sniff=9/0B)" ] }, "metrics": { "process": { "sampleCount": 347, "peakRssMb": 14756, "peakCpuPercent": 1228.6, "firstAtMs": 0, "lastAtMs": 2059392 }, "gpu": { "available": true, "sampleCount": 347, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 347, "peakUsedMb": 31277, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 84510, "totalOutputTokens": 15113, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 104668, "todayTokensCached": 98733, "todayTokensOut": 7320, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 104668, "totalTokensCached": 98733, "totalTokensOut": 7320, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 9 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3480, "outputTokens": 293, "cost": 0, "durationMs": 67822, "outputTokensPerSec": 9.090741823523732, "at": "2026-11-06T08:07:33.307Z" }, "lastUpdated": "2026-11-06T08:07:33.307Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T08:07:35.397Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2060690, "meanTokensPerSec": 8.75, "genTokensPerSec": 8.75, "genTokens": 15113, "promptTokensPerSec": 390.52, "peakRssMb": 14756, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 104668, "output": 7320 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T07:33:08.380Z", "finishedAt": "2026-11-06T08:07:38.970Z", "timeToFirstArtifactMs": 780125, "timeToLastArtifactWriteMs": 2064927, "timeToFirstTokenMs": 48227, "firstTurnTtftMs": 27701, "timeToFirstToolCallMs": 89452 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "fieldstone-launch-comms/1", "stepId": "scope", "activatedAt": "2026-11-06T07:34:37.670Z", "completedAt": "2026-11-06T07:46:08.665Z", "ms": 690995, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "d3ed61a1-c454-40f6-85bd-cf9a0a79d32f" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "draft", "activatedAt": "2026-11-06T07:46:08.666Z", "completedAt": "2026-11-06T07:48:31.815Z", "ms": 143149, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "24dd97fc-b9f9-4a3a-824b-4ff5bcf1ae27" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "review", "activatedAt": "2026-11-06T07:48:31.816Z", "completedAt": "2026-11-06T07:53:23.538Z", "ms": 291722, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "cc99ffba-14d1-4d71-8e04-231ec0ecd5b5" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "evaluate", "activatedAt": "2026-11-06T07:53:23.539Z", "completedAt": "2026-11-06T08:02:07.273Z", "ms": 523734, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "cc99ffba-14d1-4d71-8e04-231ec0ecd5b5" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "finish", "activatedAt": "2026-11-06T08:02:07.275Z", "completedAt": "2026-11-06T08:06:20.034Z", "ms": 252759, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "f98b558a-06d2-4c10-a8fe-ad99736ce136" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T07:33:14.278Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T07:33:14.165Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 291722 }, "sessions": { "total": 6, "taskScoped": 4, "byGezel": { "ines": 1, "juan": 1, "mykola": 1, "shahd": 1, "zain": 2 }, "perTask": { "fieldstone-launch-comms/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 9, "maxContextFill": 0.2747344970703125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2747344970703125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-route-multi", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "scenarioId": "craftbook-route-multi", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-06T09:17:23.937Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/workspace/fieldstone-launch-comms/out/press-release.md", "artifactBytes": 1001, "artifactSha256": "ee786d73e71aab4a898401a62d147247cfc9b29f43e2c37570080be5ba1293dd", "axisSource": "evals/src/craftbooks/authoring/route-multi.ts", "axes": [], "scoreAxes": {}, "axisJustifications": {}, "meanScore": null, "justification": "No declared axes. The requested release preserves approved facts, quote and boilerplate, with no stale facts. Five recipe stages complete, but they review a different root-path release. The requested file is rewritten after completion and lacks the recipe end mark; the earlier review does not certify it. Both divergent copies remain.", "reviewBasis": "Read final and alternate releases, approved facts, stale teaser, scope/verdict and task notes. Compared quote/boilerplate literally and traced writes, validation and task completion. No execution retry or gate change.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/workspace/fieldstone-launch-comms/out/press-release.md", "bytes": 1001, "sha256": "ee786d73e71aab4a898401a62d147247cfc9b29f43e2c37570080be5ba1293dd", "status": "reviewedNoDeclaredAxes" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/workspace/fieldstone-launch-comms/press-release.md", "bytes": 934, "sha256": "1fbc941c162ae5a5f74c0d40d8c2afde3b9052916c62cc918d0f0cd9fa8c9dc4", "status": "reviewedNoDeclaredAxes" } ], "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 22458, "maxSessionContextFill": 0.2747344970703125, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2747344970703125 }, "nativeFailureLines": [], "sampledPeaks": { "gpuMiB": 10642, "processTreeRssMiB": 14756, "wholeSystemRamMiB": 31277 }, "note": "Explicit CPU offload; shared GPU memory/paging unmeasured. Whole-system RAM includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:37.898Z", "reviewId": "082", "cellId": "r2-saluki-cuda-complex-work-craftbook-route-multi", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "scenarioId": "craftbook-route-multi", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-route-multi", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "5434f35912179b3a3a6c4fd553fefabae3d433d50d62f15ff994aa3e7443c8c7", "evidenceSha256": "1201819b8eef400e3e68124e7d9bd5fd71d280a2b43ddadfb9b2f75ddc8fff1f", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/workspace/fieldstone-launch-comms/out/press-release.md", "bytes": 1001, "sha256": "ee786d73e71aab4a898401a62d147247cfc9b29f43e2c37570080be5ba1293dd", "shownBytes": 1001, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/workspace/fieldstone-launch-comms/press-release.md", "bytes": 934, "sha256": "1fbc941c162ae5a5f74c0d40d8c2afde3b9052916c62cc918d0f0cd9fa8c9dc4", "shownBytes": 934, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The out/press-release.md file has the right date, figures, quote and contact. But it says \"has opened\" and \"Doors open on\" together, and the quote lacks a \"said\" attribution. A second, cleaner release sits at the project root, not at the required path.", "limitations": "Source facts/announcement.md and the craftbook tool-call trace were not shown. I could not verify that suggest_craftbook was invoked, that the old teaser was ignored, or that the facts match. The stray root-level file suggests an uncertain workflow. This is a source-only review.", "meanScore": null, "durationMs": 16987, "costUsd": 0.013262, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2542, "cache_read_input_tokens": 700, "output_tokens": 302, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2542, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 302, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2542, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2542 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 302, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2542, "webSearchRequests": 0, "costUSD": 0.013262, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2542, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-route-multi / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 34m 31s; budget used: 17.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 780125`; `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":995,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.17233347695912923`; `facts.toolUse.totalToolCalls: 33`; `facts.timing.timeToFirstArtifactMs: 780125`; `facts.outcome.durationMs: 2070590` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14756 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31277 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.75 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 84510 + 15113 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 292s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 27% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 780125`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":995,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.17233347695912923`\n- `facts.toolUse.totalToolCalls: 33`\n- `facts.timing.timeToFirstArtifactMs: 780125`\n- `facts.outcome.durationMs: 2070590`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 780125`; `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":995,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.17233347695912923`; `facts.toolUse.totalToolCalls: 33`; `facts.timing.timeToFirstArtifactMs: 780125`; `facts.outcome.durationMs: 2070590`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.5/10**, all 11 checks, unchanged. No declared qualitative axes; advisory **notApplicable**. Both final release files were reviewed, alongside the earlier scope and verdict.\n\nNo declared axes. The requested release preserves approved facts, quote and boilerplate, with no stale facts. Five recipe stages complete, but they review a different root-path release. The requested file is rewritten after completion and lacks the recipe end mark; the earlier review does not certify it. Both divergent copies remain.\n\nRead final and alternate releases, approved facts, stale teaser, scope/verdict and task notes. Compared quote/boilerplate literally and traced writes, validation and task completion. No execution retry or gate change.\n\n## Positives\n\n- The bundled press-release 1.0.6 recipe is selected and actually runs all five stages to completion, with four task-scoped sessions and zero gate rejections. No replacement recipe is authored (outcome.success; continuity.steps.completed=5; continuity.sessions.taskScoped=4; toolUse.byTool.start_work=1).\n- The final requested release includes the approved announcement, founder/title, location, date, 68 percent/40 km rationale, eleven employees, twice-monthly evenings, exact quote, boilerplate and contact. Neither the abandoned date nor CEO title leaks in (sniff.latest.score=11; captured final and approved facts; advisory-consistency-audit.json).\n- The editorial pass removes a redundant subhead from the root release, and the later requested-path rewrite restores the approved quote punctuation. Real read/write/edit receipts establish artifact production beyond a chat-only completion claim (toolUse.byTool.replace_lines=1; toolUse.byTool.write_file=2; advisory-tool-evidence.json).\n\n## Negatives\n\n- Recipe prompts and gates target press-release.md despite the requested out/press-release.md. The workflow completes at 08:06:20; another session writes a substantially different requested-path release at 08:07:33. The earlier verdict and DONE refer to the root file, so they do not review the final delivered bytes (continuity.steps.perStep; final task; write receipts).\n- The root release changes the approved quote final period to a comma, while the verdict and DONE call it verbatim. validate only checks nonempty content. The final rewrite restores the quote but drops the closing ### required by the completed recipe; both copies remain (toolUse.byTool.validate=1; sniff.latest score=11 reflects scenario checks, not revalidation by the recipe).\n- The final boilerplate is verbatim but precedes the contact rather than being the final paragraph requested in approved facts. The lede repeats the date and mixes has opened with doors open; these are modest editorial weaknesses despite a complete deterministic gate (outcome.success; captured final text).\n- Thirty-three calls and 34m30s include coordination across five stages plus post-completion output repair. The recipe description allows a supplied destination but the captured prompts/gates use a literal path (toolUse.totalToolCalls=33; outcome.durationMs=2070590; final task craftbook).\n\n## Performance and GPU fit\n\n**34m30s**, **33 calls**, **15,113 engine-generated tokens at 8.75 tok/s**, prefill **390.52 tok/s** (outcome.durationMs; toolUse.totalToolCalls; perf.derived). Sampled peak GPU **10,642/12,227 MiB**, process-tree RSS **14,756 MiB**, whole-system RAM **31,277/31,865 MiB**. Whole-system RAM includes other applications; shared GPU memory/paging unmeasured. No new campaign peak.\n\nOne native 0.1.57 CUDA launch, 65,536 context, one slot and f16 KV. Dense FFN placement: 39/64 on GPU, 25 on CPU. Maximum session fill **27.47%**; largest released native slot **22458 tokens (34.3%)**. No compaction, native endpoint failure, explicit OOM or context overflow observed. This continues to demonstrate hybrid execution within available VRAM, not full GPU residency.\n\n## Tactical experiments — not applied\n\nIn ../gilde/data/craftbook-templates/pr/press-release (source for captured version 1.0.6), introduce a destination parameter defaulting to press-release.md and use the resolved value consistently in draft/review/finish prompts and file gates. Bind a user-supplied destination through start_work parameters before the first stage. Measure duplicate outputs and post-completion repair calls, rather than teaching this scenario's out/ path to a global prompt.\n\nIn the recipe reviewer prompt, require comparison of approved quotation text including punctuation when the source explicitly says verbatim; distinguish structural validation from fact/quote validation. Measure unsupported verification claims against exact source excerpts. No sampling change is supported by this single output-path failure.\n\n## Strategic experiments — not applied\n\nIn packages/service/src/tasks, bind verdict receipts to the deliverable path and content hash, and invalidate their certification if a subsequent write changes the deliverable. Expose the canonical output location to the finish stage so a completed workflow cannot silently certify one file while a follow-up supplies another. Apply the rule to arbitrary workspace/artifact outputs, not just press releases.\n\nIn evals/src/craftbooks/authoring/route-multi.ts, a future separate evaluator improvement should verify that the reviewed/finished task artifact is the requested delivered artifact, and that exact approved quotes are preserved. Keep this completed run's score and gates unchanged. Separate recipe execution, correct final content and review lineage in reporting.\n\n## Generalization and next experiment\n\nR1 also passed fixed 8.5 with the same destination mismatch, but its requested and root copies were identical. R2 leaves divergent copies and writes the requested one after task completion. This is recurring path-binding behavior with repetition variability under unchanged native/tuning identity, not evidence of a native-version change.\n\nUnrelated invoice-run completes its recipe while leaving the ledger unchanged, and author-linear has unenforced semantic/negative-verdict checks. Together these show that workflow completion and a good-looking artifact need independent validation against the user's final deliverable contract. Test canonical destination binding and content-hash review receipts across communications, invoicing and data cleanup. Measure stale verdicts, duplicate outputs, false passes and total repair time; no fixture-specific wording or expected values should enter global model guidance.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/facts.json): pass, fixed 8.5, 32.31 min, 8.21 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/facts.json): fail, fixed 5.5, 5.05 min, 99.49 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-40-55-967Z-9xzl/facts.json): pass, fixed 8.5, 10.49 min, 114.39 tok/s.\n\nOther-model history is context, not a controlled native-version comparison.\n\n## Strategic note\n\nThe run demonstrates real library routing, multi-role workflow execution and a factually faithful final release. It also repeats a recipe destination defect and weakens final review provenance. The fixed pass should not be interpreted as proof that the exact delivered file passed the recipe's editorial and finish checks. No source, gates, prompts, tuning or retries were changed.\n\n## Raw evidence\n\nfacts.json, score.json, state.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json, advisory-progress-2026-11-06T0802 and captured workspace.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-export-generalize", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-export-generalize", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T08:10:28.853Z", "finishedAt": "2026-11-06T14:51:19.073Z", "durationMs": 24050220, "success": false, "reason": "hit hard ceiling (12015019ms + 200m progress extensions) — last hard progress 286s before the end; the deliverable never closed", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "failureMode": "timeout", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-export-generalize", "score": 5, "bytes": 3875, "deliverableMissing": false, "milestones": 1, "failReason": "task supplier-intake/3 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "timeout" }, { "source": "facts", "path": "outcome.reason", "value": "hit hard ceiling (12015019ms + 200m progress extensions) — last hard progress 286s before the end; the deliverable never closed" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2533888 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 3875, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/3 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } } ] }, "quality": { "score": 8.3, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 83.3% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2533888 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 3875, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/3 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } } ] }, "efficiency": { "score": 0, "ruleId": "E_BOTH_SEVERE", "summary": "The trial exhausted its budget while making at least thirty tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 206 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2533888 }, { "source": "facts", "path": "outcome.durationMs", "value": 24050220 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 4.8, "band": "framework-gap" }, "facts": { "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 4047, "peakRssMb": 14534, "peakCpuPercent": 1290.6, "firstAtMs": 1, "lastAtMs": 24032467 }, "gpu": { "available": true, "sampleCount": 4047, "peakUtilPercent": 100, "peakMemUsedMb": 10650, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 4047, "peakUsedMb": 30584, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 969086, "totalOutputTokens": 164291, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 28, "todayTokensIn": 592112, "todayTokensCached": 355342, "todayTokensOut": 13834, "todayCost": 0, "totalTurns": 28, "totalTokensIn": 592112, "totalTokensCached": 355342, "totalTokensOut": 13834, "totalCost": 0, "medianOutputTokensPerSec": 8.1, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.1, "turns": 28 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 23551, "outputTokens": 722, "cost": 0, "durationMs": 678958, "outputTokensPerSec": 8.179787694688166, "at": "2026-11-06T14:32:45.587Z" }, "lastUpdated": "2026-11-06T14:32:45.587Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T14:51:09.266Z" } }, "source": "engine-log" }, "derived": { "durationMs": 24034367, "meanTokensPerSec": 8.07, "genTokensPerSec": 8.07, "genTokens": 164291, "promptTokensPerSec": 353.06, "peakRssMb": 14534, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 592112, "output": 13834 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 5 }, "steps": { "activated": 9, "activationEvents": 7, "completed": 4, "gateApprovals": 4, "gateRejections": 3, "redrives": 0, "perStep": [ { "ref": "supplier-intake/1", "stepId": "build", "activatedAt": "2026-11-06T10:52:44.304Z", "completedAt": "2026-11-06T11:11:08.742Z", "ms": 1104438, "gateRejections": 1, "toolCalls": 26, "sessionIds": [ "ab5009c3-a0a7-462f-993a-e94f2b8b6371" ] }, { "ref": "supplier-intake/1", "stepId": "evaluate", "activatedAt": "2026-11-06T11:11:08.744Z", "completedAt": "2026-11-06T11:21:33.106Z", "ms": 624362, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "bd457093-6b4a-411b-a99d-d2819ddc677c" ] }, { "ref": "supplier-intake/1", "stepId": "finish", "activatedAt": "2026-11-06T11:21:33.108Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "7ed7249e-f321-4ca6-aae3-0cb5df69c279" ] }, { "ref": "supplier-intake/2", "stepId": "build", "activatedAt": "2026-11-06T11:51:41.394Z", "completedAt": "2026-11-06T12:39:17.504Z", "ms": 2856110, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "51311715-c8b4-4cc3-a16f-f89469079948" ] }, { "ref": "supplier-intake/2", "stepId": "evaluate", "activatedAt": "2026-11-06T12:39:17.506Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 19, "sessionIds": [ "f3d166dc-31d4-41ee-a102-2c920b32c692" ] }, { "ref": "supplier-intake/3", "stepId": "1", "activatedAt": "2026-11-06T13:06:17.872Z", "completedAt": "2026-11-06T13:30:51.487Z", "ms": 1473615, "gateRejections": 2, "toolCalls": 20, "sessionIds": [ "13f48c12-a45b-4a33-a30d-309cfa12e50e" ] }, { "ref": "supplier-intake/3", "stepId": "2", "activatedAt": "2026-11-06T13:30:51.489Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T08:10:34.746Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T08:10:34.665Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1289026.5 }, "sessions": { "total": 11, "taskScoped": 6, "byGezel": { "fiona": 2, "imara": 1, "imre": 5, "kofi": 2, "reza": 1 }, "perTask": { "supplier-intake/3": 1, "supplier-intake/2": 2, "supplier-intake/1": 3 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6666666666666666, "maxMessages": 18, "maxContextFill": 0.31695556640625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.31695556640625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 2 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 3, "tasks": 5, "tasksWithReferences": 3, "items": 15, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/quarterly-summary", "knowledge://bendyline/handboek/craftbook/reorder-run", "knowledge://bendyline/handboek/craftbook/recipe-capture", "knowledge://bendyline/handboek/craftbook/cookbook-layout", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/receipt-intake", "knowledge://bendyline/handboek/craftbook/recipe-capture", "knowledge://bendyline/handboek/craftbook/reorder-run", "knowledge://bendyline/handboek/craftbook/build-loop", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/receipt-intake", "knowledge://bendyline/handboek/craftbook/recipe-capture", "knowledge://bendyline/handboek/craftbook/reorder-run", "knowledge://bendyline/handboek/craftbook/build-loop" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "timeout", "reason": "hit hard ceiling (12015019ms + 200m progress extensions) — last hard progress 286s before the end; the deliverable never closed", "durationMs": 24050220, "timeoutMs": 12015019, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-06T08:10:28.853Z", "finishedAt": "2026-11-06T14:51:19.073Z", "timeToFirstArtifactMs": 2533888, "timeToLastArtifactWriteMs": 22255279, "timeToFirstTokenMs": 47680, "firstTurnTtftMs": 36815, "timeToFirstToolCallMs": 55997 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer", "Data-Analyst", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 206, "byTool": { "read_file": 40, "list_dir": 15, "list_craftbooks": 1, "list_project_gezels": 1, "craftbook_read": 8, "ensure_gezel": 1, "message_gezel": 4, "craftbook_write": 5, "invoke_craftbook": 1, "list_tasks": 4, "consult_developer": 2, "start_work": 11, "suggest_work": 1, "get_task": 3, "read_task_notes": 12, "list_artifacts": 9, "list_projects": 1, "read_artifact": 7, "delegate_developer": 1, "read_files": 7, "search_memory": 1, "find_files": 1, "write_artifact": 3, "write_file": 19, "how_do_i": 2, "stat": 9, "validate": 16, "save_memory": 1, "write_task_note": 10, "replace_in_file": 1, "advance_task_step": 6, "set_task_status": 1, "run_installed_script": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/supplier-intake/index.html", "finalBytes": 5380, "growth": [] } ], "imageFiles": [], "otherFileCount": 13 }, "sniff": { "progression": [ { "atMs": 7905, "filePath": "craftbook-export-generalize", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 4269988, "filePath": "craftbook-export-generalize", "bytes": 19, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 4412393, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 8049814, "filePath": "craftbook-export-generalize", "bytes": 1863, "score": 3, "scoreMax": 6, "signals": [], "failReason": "no task has been created from craftbook \"supplier-intake\" yet — invoke it on this project so the Q2 pass is produced BY the recipe" }, { "atMs": 9022867, "filePath": "craftbook-export-generalize", "bytes": 3641, "score": 5, "scoreMax": 6, "signals": [], "failReason": "no task has been created from craftbook \"supplier-intake\" yet — invoke it on this project so the Q2 pass is produced BY the recipe" }, { "atMs": 11338417, "filePath": "craftbook-export-generalize", "bytes": 3675, "score": 5, "scoreMax": 6, "signals": [], "failReason": "no task has been created from craftbook \"supplier-intake\" yet — invoke it on this project so the Q2 pass is produced BY the recipe" }, { "atMs": 15085331, "filePath": "craftbook-export-generalize", "bytes": 3667, "score": 5, "scoreMax": 6, "signals": [], "failReason": "no task has been created from craftbook \"supplier-intake\" yet — invoke it on this project so the Q2 pass is produced BY the recipe" }, { "atMs": 16099573, "filePath": "craftbook-export-generalize", "bytes": 3667, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/3 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 16895319, "filePath": "craftbook-export-generalize", "bytes": 4110, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/3 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 21157177, "filePath": "craftbook-export-generalize", "bytes": 4197, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/3 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 21163484, "filePath": "craftbook-export-generalize", "bytes": 3875, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/3 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } ], "latest": { "filePath": "craftbook-export-generalize", "bytes": 3875, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/3 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 12383146, "kind": "structured", "gezel": "imara", "question": "Your web page is finished. - **`index.html`** - `tasks/1/verdict.md` - `out/q2-…", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-06T08:10:28.857Z] [trial] id=craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3 scenario=craftbook-export-generalize model=underdog-saluki-27b-iq2-mix", "[2026-11-06T08:10:28.857Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-export-generalize-FOd4Re", "[2026-11-06T08:10:28.857Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "[2026-11-06T08:10:28.857Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T08:10:28.860Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T08:10:28.857Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T08:10:28.865Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T08:10:28.871Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-export-generalize-FOd4Re", "[2026-11-06T08:10:28.871Z] [trial] throughput-scaled ceiling: 80m → 200m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T08:10:34.775Z] [trial] daemon spawned pid=45708 port=https://127.0.0.1:50264", "[2026-11-06T08:10:36.122Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T08:10:36.140Z] [trial] meester=imara", "[2026-11-06T08:10:36.140Z] [trial] running scenario setup hook", "[2026-11-06T08:10:36.504Z] [trial] scenario setup complete", "[2026-11-06T08:10:36.507Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T08:10:36.585Z] [poll] initial digests hard=230d3404b35d6de1 soft=c6e75b56b79f2066", "[2026-11-06T08:10:36.866Z] [poll] hard-progress: 230d3404b35d6de1 → cd19b20fd0602fe2 after 0s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-06T08:11:29.619Z] [poll] hard-progress: cd19b20fd0602fe2 → 7fda5de378fb58a3 after 53s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=39,stream=75) sniff=1/0B)", "[2026-11-06T08:19:52.510Z] [poll] hard-progress: 7fda5de378fb58a3 → 1610a92b399c8c7b after 503s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=1398,stream=4121) sniff=1/0B)", "[2026-11-06T08:29:10.855Z] [poll] hard-progress: 1610a92b399c8c7b → 3dbb1ce7c1c6e709 after 558s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=2786,stream=8238) sniff=1/0B)", "[2026-11-06T08:29:40.309Z] [poll] hard-progress: 3dbb1ce7c1c6e709 → 00d91d79d90b0b34 after 29s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=2802,stream=8282) sniff=1/0B)", "[2026-11-06T08:38:28.847Z] [poll] hard-progress: 00d91d79d90b0b34 → 3f7409cde7a990ad after 529s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=4169,stream=12321) sniff=1/0B)", "[2026-11-06T08:45:25.351Z] [poll] hard-progress: 3f7409cde7a990ad → 33224799083f7828 after 417s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=9,slot=5266,stream=15755) sniff=1/0B)", "[2026-11-06T08:46:18.960Z] [poll] hard-progress: 33224799083f7828 → 5173d4b010f04459 after 54s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=10,slot=5305,stream=15884) sniff=1/0B)", "[2026-11-06T08:46:30.725Z] [poll] hard-progress: 5173d4b010f04459 → f5b9b20af1a8256b after 12s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=11,slot=5327,stream=15939) sniff=1/0B)" ] }, "metrics": { "process": { "sampleCount": 4047, "peakRssMb": 14534, "peakCpuPercent": 1290.6, "firstAtMs": 1, "lastAtMs": 24032467 }, "gpu": { "available": true, "sampleCount": 4047, "peakUtilPercent": 100, "peakMemUsedMb": 10650, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 4047, "peakUsedMb": 30584, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 969086, "totalOutputTokens": 164291, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 28, "todayTokensIn": 592112, "todayTokensCached": 355342, "todayTokensOut": 13834, "todayCost": 0, "totalTurns": 28, "totalTokensIn": 592112, "totalTokensCached": 355342, "totalTokensOut": 13834, "totalCost": 0, "medianOutputTokensPerSec": 8.1, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.1, "turns": 28 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 23551, "outputTokens": 722, "cost": 0, "durationMs": 678958, "outputTokensPerSec": 8.179787694688166, "at": "2026-11-06T14:32:45.587Z" }, "lastUpdated": "2026-11-06T14:32:45.587Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T14:51:09.266Z" } }, "source": "engine-log" }, "derived": { "durationMs": 24034367, "meanTokensPerSec": 8.07, "genTokensPerSec": 8.07, "genTokens": 164291, "promptTokensPerSec": 353.06, "peakRssMb": 14534, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 592112, "output": 13834 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T08:10:28.853Z", "finishedAt": "2026-11-06T14:51:19.073Z", "timeToFirstArtifactMs": 2533888, "timeToLastArtifactWriteMs": 22255279, "timeToFirstTokenMs": 47680, "firstTurnTtftMs": 36815, "timeToFirstToolCallMs": 55997 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 5 }, "steps": { "activated": 9, "activationEvents": 7, "completed": 4, "gateApprovals": 4, "gateRejections": 3, "redrives": 0, "perStep": [ { "ref": "supplier-intake/1", "stepId": "build", "activatedAt": "2026-11-06T10:52:44.304Z", "completedAt": "2026-11-06T11:11:08.742Z", "ms": 1104438, "gateRejections": 1, "toolCalls": 26, "sessionIds": [ "ab5009c3-a0a7-462f-993a-e94f2b8b6371" ] }, { "ref": "supplier-intake/1", "stepId": "evaluate", "activatedAt": "2026-11-06T11:11:08.744Z", "completedAt": "2026-11-06T11:21:33.106Z", "ms": 624362, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "bd457093-6b4a-411b-a99d-d2819ddc677c" ] }, { "ref": "supplier-intake/1", "stepId": "finish", "activatedAt": "2026-11-06T11:21:33.108Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "7ed7249e-f321-4ca6-aae3-0cb5df69c279" ] }, { "ref": "supplier-intake/2", "stepId": "build", "activatedAt": "2026-11-06T11:51:41.394Z", "completedAt": "2026-11-06T12:39:17.504Z", "ms": 2856110, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "51311715-c8b4-4cc3-a16f-f89469079948" ] }, { "ref": "supplier-intake/2", "stepId": "evaluate", "activatedAt": "2026-11-06T12:39:17.506Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 19, "sessionIds": [ "f3d166dc-31d4-41ee-a102-2c920b32c692" ] }, { "ref": "supplier-intake/3", "stepId": "1", "activatedAt": "2026-11-06T13:06:17.872Z", "completedAt": "2026-11-06T13:30:51.487Z", "ms": 1473615, "gateRejections": 2, "toolCalls": 20, "sessionIds": [ "13f48c12-a45b-4a33-a30d-309cfa12e50e" ] }, { "ref": "supplier-intake/3", "stepId": "2", "activatedAt": "2026-11-06T13:30:51.489Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T08:10:34.746Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T08:10:34.665Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1289026.5 }, "sessions": { "total": 11, "taskScoped": 6, "byGezel": { "fiona": 2, "imara": 1, "imre": 5, "kofi": 2, "reza": 1 }, "perTask": { "supplier-intake/3": 1, "supplier-intake/2": 2, "supplier-intake/1": 3 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6666666666666666, "maxMessages": 18, "maxContextFill": 0.31695556640625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.31695556640625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 2 } }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-export-generalize", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "scenarioId": "craftbook-export-generalize", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-06T15:38:09.914Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q1-intake.md", "artifactBytes": 837, "artifactSha256": "472605a072228a3bb79db0abc8ff74da67ea12d34a0478d3600aee4168affd4c", "axisSource": "evals/src/craftbooks/authoring/export-generalize.ts", "axes": [], "scoreAxes": {}, "axisJustifications": {}, "meanScore": null, "justification": "No declared axes. All four requested files agree on seven faithful nonnegative records per quarter. The invoked recipe uses a conflicting positive-only rule and literal {quarter} paths; its second step never executes before timeout. Script calls fail, direct writes substitute, and supporting reviews become stale. Loading succeeds; workflow generalization remains incomplete.", "reviewBasis": "Read final requested reports/arrays and source fixtures; recomputed every retained field and ordering. Verified hashes of all supporting outputs against the prior full reviews. Traced recipe versions, writes, rejected gates, script failures and final task state across six preserved progress snapshots.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q1-intake.md", "bytes": 837, "sha256": "472605a072228a3bb79db0abc8ff74da67ea12d34a0478d3600aee4168affd4c", "role": "declaredOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q1-clean.json", "bytes": 1102, "sha256": "5afd1ac820f6351024084c81a0821378c962dbc520a5a0c90b987645f229138a", "role": "declaredOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q2-intake.md", "bytes": 833, "sha256": "07fa6968853146aa0b7eb518820b6d659091747e2c72bf9b4c4fd9c86ce26fdc", "role": "declaredOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q2-clean.json", "bytes": 1106, "sha256": "bb337c953adb23147f6ed058000f57ac7dce3620b00aeef7aed22a04e169e20c", "role": "declaredOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/scripts/derive.mjs", "bytes": 2324, "sha256": "de4be8a1a03802fd062777c7cfa254b28aca414b1c54669fd8f9ba0362ab28d8", "role": "supportingOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/scripts/intake.mjs", "bytes": 2858, "sha256": "a1c3e9d812878ba494308147fea7bfb181df07e8941bb8bc253ba6027d46f08b", "role": "supportingOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/index.html", "bytes": 5380, "sha256": "5958de641e0517918ffd4e1bc486eb7084a5854991f937fa7485d05e073602a0", "role": "supportingOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/report.md", "bytes": 2686, "sha256": "dbc6d2d5916db8386b7845c4e0bdfa10e763688db3c813773a026c1b30659cfd", "role": "supportingOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/{quarter}-intake.md", "bytes": 1155, "sha256": "312c2f5b6f60f425edb28dc2b37808e5f0c47ef68ebd37fb5a33cd24af4da27f", "role": "supportingOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/{quarter}-clean.json", "bytes": 0, "role": "authoredGateTarget", "status": "noArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/artifacts/supplier-intake/reports/supplier-intake-report.md", "bytes": 2843, "sha256": "3e58d1cbd4a15e7db1af2474948308ef4f7245d870edc8b68b8f3e22603b1e45", "role": "supportingOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/artifacts/supplier-intake/tasks/1/verdict.md", "bytes": 912, "sha256": "d43f1ba868d0138992a3061dfcc0afb4456e7900b207c29384c5140e160d1a07", "role": "supportingOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/artifacts/supplier-intake/tasks/2/verdict.md", "bytes": 0, "role": "supportingOutput", "status": "noArtifact" } ], "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 44401, "maxSessionContextFill": 0.31695556640625, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.31695556640625 }, "nativeFailureLines": [], "sampledPeaks": { "gpuMiB": 10650, "processTreeRssMiB": 14534, "wholeSystemRamMiB": 30584 }, "note": "Hybrid placement, not full GPU residency. Shared GPU memory and paging unmeasured; whole-system RAM includes other apps." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:39.542Z", "reviewId": "083", "cellId": "r2-saluki-cuda-complex-work-craftbook-export-generalize", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "scenarioId": "craftbook-export-generalize", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-export-generalize", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "b0924f790e9ba1fe2d936e40a350016db690b1b21a66cffa12ea13a5113568ce", "evidenceSha256": "41ff2cfdbad6fe90bd3ed5215d863738db69eba32e407b4639b80df22aa26d1a", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q1-intake.md", "bytes": 837, "sha256": "472605a072228a3bb79db0abc8ff74da67ea12d34a0478d3600aee4168affd4c", "shownBytes": 837, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q1-clean.json", "bytes": 1102, "sha256": "5afd1ac820f6351024084c81a0821378c962dbc520a5a0c90b987645f229138a", "shownBytes": 1102, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q2-intake.md", "bytes": 833, "sha256": "07fa6968853146aa0b7eb518820b6d659091747e2c72bf9b4c4fd9c86ce26fdc", "shownBytes": 833, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q2-clean.json", "bytes": 1106, "sha256": "bb337c953adb23147f6ed058000f57ac7dce3620b00aeef7aed22a04e169e20c", "shownBytes": 1106, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/scripts/derive.mjs", "bytes": 2324, "sha256": "de4be8a1a03802fd062777c7cfa254b28aca414b1c54669fd8f9ba0362ab28d8", "shownBytes": 2324, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/scripts/intake.mjs", "bytes": 2858, "sha256": "a1c3e9d812878ba494308147fea7bfb181df07e8941bb8bc253ba6027d46f08b", "shownBytes": 2858, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/index.html", "bytes": 5380, "sha256": "5958de641e0517918ffd4e1bc486eb7084a5854991f937fa7485d05e073602a0", "shownBytes": 5380, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/report.md", "bytes": 2686, "sha256": "dbc6d2d5916db8386b7845c4e0bdfa10e763688db3c813773a026c1b30659cfd", "shownBytes": 2686, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/{quarter}-intake.md", "bytes": 1155, "sha256": "312c2f5b6f60f425edb28dc2b37808e5f0c47ef68ebd37fb5a33cd24af4da27f", "shownBytes": 1155, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/artifacts/supplier-intake/reports/supplier-intake-report.md", "bytes": 2843, "sha256": "3e58d1cbd4a15e7db1af2474948308ef4f7245d870edc8b68b8f3e22603b1e45", "shownBytes": 2843, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/artifacts/supplier-intake/tasks/1/verdict.md", "bytes": 912, "sha256": "d43f1ba868d0138992a3061dfcc0afb4456e7900b207c29384c5140e160d1a07", "shownBytes": 912, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3\\workspace\\supplier-intake\\data\\suppliers-q1.json", "bytes": 1260, "sha256": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "shownBytes": 1260, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3\\workspace\\supplier-intake\\data\\suppliers-q2.json", "bytes": 1263, "sha256": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "shownBytes": 1263, "truncated": false } ], "scoreAxes": {}, "justification": "Qualitative: Q1/Q2 outputs are consistent (flag only negatives, 7 clean each). But no craftbook artifact is shown, only scripts/intake.mjs; the \"{quarter}\" literal path files and stale report.md (flagging 0-day) show a muddled run. Recipe-save and invoke_craftbook can't be verified.", "limitations": "Evidence has no craftbook_write/invoke_craftbook traces or saved library book, so it can't confirm Q2 came from the recipe rather than a script. Runner-terminal with missing paths. Stray files with conflicting rules (0 flagged) leave the final state ambiguous.", "meanScore": null, "durationMs": 15022, "costUsd": 0.06742, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 15999, "cache_read_input_tokens": 700, "output_tokens": 335, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 15999, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 335, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 15999, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 15999 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 335, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 15999, "webSearchRequests": 0, "costUSD": 0.06742, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 15999, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-export-generalize / underdog-saluki-27b-iq2-mix\n\n**Composite: 4.8 / 10** (band: framework-gap)\n**Outcome: timeout** (duration: 400m 50s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (12015019ms + 200m progress extensions) — last hard progress 286s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 2533888`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3875,\"score\":5,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task supplier-intake/3 (from craftbook \\\"supplier-intake\\\") has status \\...` |\n| Output quality (25%) | 8.3 / 10 | The latest scenario gate reached 83.3% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 2533888`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3875,\"score\":5,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task supplier-intake/3 (from craftbook \\\"supplier-intake\\\") has status \\...` |\n| Process efficiency (20%) | 0 / 10 | The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 206`; `facts.timing.timeToFirstArtifactMs: 2533888`; `facts.outcome.durationMs: 24050220` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 3 + 0.25 * 8.3 + 0.2 * 0 + 0.15 * 10 = **4.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14534 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10650 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30584 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.07 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 969086 + 164291 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 5s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 9 → 4 | `continuity.steps` |\n| Gate rejections / redrives | 3 / 0 | `continuity.steps` |\n| Median step wall-clock | 1289s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 6 / 0.67 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 1 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 32% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 2 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 3 / 3 of 5 / 15 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"timeout\"`\n- `facts.outcome.reason: \"hit hard ceiling (12015019ms + 200m progress extensions) — last hard progress 286s before the end; the deliverable never closed\"`\n- `facts.timing.timeToFirstArtifactMs: 2533888`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3875,\"score\":5,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task supplier-intake/3 (from craftbook \\\"supplier-intake\\\") has status \\...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 83.3% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 2533888`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3875,\"score\":5,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task supplier-intake/3 (from craftbook \\\"supplier-intake\\\") has status \\...`\n\n### Process efficiency - E_BOTH_SEVERE\n\nThe trial exhausted its budget while making at least thirty tool calls.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 206`\n- `facts.timing.timeToFirstArtifactMs: 2533888`\n- `facts.outcome.durationMs: 24050220`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (12015019ms + 200m progress extensions) — last hard progress 286s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 2533888`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3875,\"score\":5,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task supplier-intake/3 (from craftbook \\\"supplier-intake\\\") has status \\...`\n- Quality: The latest scenario gate reached 83.3% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 2533888`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3875,\"score\":5,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task supplier-intake/3 (from craftbook \\\"supplier-intake\\\") has status \\...`\n- Efficiency: The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 206`; `facts.timing.timeToFirstArtifactMs: 2533888`; `facts.outcome.durationMs: 24050220`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **FAIL 4.8/10**, 5/6 checks, unchanged. No declared qualitative axes: **notApplicable**, with absent supporting outputs individually marked **noArtifact**. No qualitative scores were invented.\n\nNo declared axes. All four requested files agree on seven faithful nonnegative records per quarter. The invoked recipe uses a conflicting positive-only rule and literal {quarter} paths; its second step never executes before timeout. Script calls fail, direct writes substitute, and supporting reviews become stale. Loading succeeds; workflow generalization remains incomplete.\n\nRead final requested reports/arrays and source fixtures; recomputed every retained field and ordering. Verified hashes of all supporting outputs against the prior full reviews. Traced recipe versions, writes, rejected gates, script failures and final task state across six preserved progress snapshots.\n\n## Positives\n\n- All four requested deliverables exist: two genuine quarter-specific reports and two arrays preserving every field/order of the seven nonnegative source records. Both reports correctly agree with their arrays (sniff.latest 5/6, bytes=3875; advisory-consistency-audit.comparisons).\n- A reusable two-step recipe is eventually authored and invoked; its first stage actually advances, rather than only being described in chat (continuity.steps.perStep supplier-intake/3 step 1 completed; final state task craftbook/version).\n- Native inference continues through 164,291 generated tokens with one successful mid-loop compaction and no detected native/OOM/context failure (perf.derived; continuity.compaction.midTurn=1, compactFailed=0; daemon log).\n\n## Negatives\n\n- The actual recipe never completes: step 2 has zero tool calls and no task-scoped session; the trial hits its standard hard ceiling after 400m50s (outcome.failureMode=timeout; continuity.steps.perStep; sniff.latest.failReason).\n- Single-brace template paths remain literal. Two gate rejections lead to a duplicate out/{quarter}-intake.md rather than resolved destination repair; the corresponding clean JSON is absent (continuity.steps.gateRejections=3 total including generic HTML rejection; state.tasks; captured files).\n- Competing sessions apply different zero-lead policies and overwrite shared outputs, invalidating prior review. The final requested files are internally consistent but differ from the invoked positive-only recipe; the user-facing zero-day policy is ambiguous (toolUse.totalToolCalls=206; write chronology; final task prompts; source fixture).\n- Two installed-script attempts fail, while notes claim the Q2 pass was generated by running the recipe/script. Generic build-loop adds an irrelevant HTML requirement and another unfinished review task (toolUse.byTool; final state; script receipts).\n- Evidence guarding rejects an earlier derived-count claim even after source reading, but later correctly blocks a stale six-row verdict after the output changes. Tool guidance, stale reads and shared write ownership all contribute; do not treat every guard rejection as a false positive (toolUse and recording receipts; continuity.steps.gateRejections).\n\n## Performance and GPU fit\n\n**400m50s**, **206 tool calls**, **164,291 engine-generated tokens at 8.07 tok/s**, prefill **353.06 tok/s** (outcome.durationMs, toolUse.totalToolCalls, perf.derived). Sampled peak GPU **10650/12227 MiB**, process-tree RSS **14534 MiB**, whole-system RAM **30584/31865 MiB**. Whole-system RAM includes other applications; shared GPU memory and paging are unmeasured. No new campaign peak.\n\nOne native 0.1.57 CUDA launch; **65,536-token context**, one slot, f16 KV, **39/64 dense FFN layers on GPU and 25 on CPU**. Largest released native slot **44401 tokens (67.8%)**; maximum recorded session fill **31.70%**. One successful mid-loop compaction, zero compaction failures. No native endpoint failure, explicit OOM or context overflow found. These observations demonstrate hybrid execution within VRAM, not full GPU residency or absence of unmeasured shared-memory paging.\n\n## Tactical experiments — not applied\n\nIn packages/mcp/src/installed-script-miss.ts, replace the unconditional advice to write the output directly with method-aware guidance: when the request requires running a script, report installed versus workspace-script execution distinctly and expose an available supported execution/delegation route. Require an execution receipt before a completion note may say a script ran. The two failed calls followed by direct writes justify this experiment; do not hardcode these filenames.\n\nIn the generic build-loop template under ../gilde/data/craftbook-templates, add an explicit output-medium/destination parameter and select its gate from that contract instead of requiring index.html for data work. In the recipe-authoring guidance, require a declared parameter schema and double-brace interpolation for parameters used in prompts/gates. Validate one resolved invocation before task dispatch. No sampling or token-budget change is supported by this particular failure.\n\n## Strategic experiments — not applied\n\nAt craftbook validation/invocation (packages/core/src/schemas/craftbook.ts and packages/service/src/tasks/task-creation.ts), diagnose template-like but unresolved paths before creating a task; return the invalid field plus supported token syntax instead of allowing a file-size gate on a literal placeholder. Preserve valid literal filenames through explicit escaping rather than banning braces universally.\n\nIn packages/service/src/tasks/step-runtime.ts and writes-off-deliverables.ts, bind a task's writable deliverables and review receipts to content hashes and a versioned validation policy. Reject stale review certification and coordinate writers from parallel tasks. The Q1 approval race and final Q2 stale rejection provide concrete evidence. Investigate the second-stage lack of a dispatched session independently rather than inferring a cause solely from the timeout.\n\nIn packages/core/src/checks/claims.ts, distinguish reproducible derived aggregates from unsupported numeric assertions, while retaining the guard's correct rejection of stale content. Measure false rejections and stale-claim escapes together.\n\n## Generalization and next experiment\n\nR1 failed fixed 2.4 with no requested artifacts, an uninvoked recipe and authoring/repetition failures. R2 improves to fixed 4.8 and faithful requested data files, yet still does not demonstrate the reusable workflow completing. This is variation under frozen settings, not a native-version improvement claim.\n\nThe following author-params trial uses proper double-brace tokens, separate task-scoped intermediate files and two successful invocations; its final arithmetic is correct. It nevertheless briefly permits an incorrect concurrent write and finishes evaluation before the south task is terminal. Unrelated invoice-run and route-multi likewise separate plausible final outputs from completed/reviewed workflow contracts. Evaluate parameter validation, output ownership and hash-bound review across data cleanup, sales aggregation, communications and invoice updates; measure false passes, stale reviews and task closure, with no fixture-specific global prompts.\n\n## Strategic note\n\nStable hybrid inference over nearly seven hours is evidence of sustained native operation, not proof of reliable multi-task generalization. The failure is concretely traceable to recipe authoring/invocation, literal paths, competing policies, unsupported execution claims and incomplete task dispatch/closure. No capability retry, budget change, source change or tuning was applied.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/facts.json): fail, fixed 2.4, 69.62 min, 8.32 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/facts.json): fail, fixed 3.3, 13.63 min, 103.51 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-09-16-913Z-wk3n/facts.json): fail, fixed 2.4, 6.19 min, 102.98 tok/s.\n\nOnly same-campaign repetitions share the frozen comparison identity. Other-model history is contextual, not a controlled native-version comparison.\n\n## Raw evidence\n\nfacts.json, score.json, state.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json, recording/chat-events.jsonl and captured workspace/artifacts. Six earlier advisory-progress snapshots preserve the overwritten outputs and evolving recipe.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-author-params", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-author-params", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T14:54:26.200Z", "finishedAt": "2026-11-06T15:13:34.660Z", "durationMs": 1148460, "success": true, "reason": "authored one parameterized craftbook, ran it twice with different region inputs, and both region summaries exist with distinct contents", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 697837 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 216, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08496477297746464 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 37 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 697837 }, { "source": "facts", "path": "outcome.durationMs", "value": 1148460 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 193, "peakRssMb": 15354, "peakCpuPercent": 1208.6, "firstAtMs": 1, "lastAtMs": 1136878 }, "gpu": { "available": true, "sampleCount": 193, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 193, "peakUsedMb": 31150, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 86644, "totalOutputTokens": 6773, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 57534, "todayTokensCached": 53377, "todayTokensOut": 1894, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 57534, "totalTokensCached": 53377, "totalTokensOut": 1894, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3347, "outputTokens": 70, "cost": 0, "durationMs": 90525, "outputTokensPerSec": 8.663258030369365, "at": "2026-11-06T15:13:19.277Z" }, "lastUpdated": "2026-11-06T15:13:19.277Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T15:13:30.768Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1138370, "meanTokensPerSec": 8.51, "genTokensPerSec": 8.51, "genTokens": 6773, "promptTokensPerSec": 361.7, "peakRssMb": 15354, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 57534, "output": 1894 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 4 }, "steps": { "activated": 6, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "regional-sales-rollup/1", "stepId": "compute", "activatedAt": "2026-11-06T15:01:52.212Z", "completedAt": "2026-11-06T15:06:04.231Z", "ms": 252019, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "908d7d74-7076-41c3-b9fe-77b73aa31d8d" ] }, { "ref": "regional-sales-rollup/1", "stepId": "write-summary", "activatedAt": "2026-11-06T15:06:04.232Z", "completedAt": "2026-11-06T15:12:29.930Z", "ms": 385698, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "0a004f5f-760b-4c1e-a015-6703207f1a48" ] }, { "ref": "regional-sales-rollup/2", "stepId": "compute", "activatedAt": "2026-11-06T15:01:52.607Z", "completedAt": "2026-11-06T15:09:49.283Z", "ms": 476676, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "f374bafb-a42d-4c6d-8965-d6356d668ae7" ] }, { "ref": "regional-sales-rollup/2", "stepId": "write-summary", "activatedAt": "2026-11-06T15:09:49.284Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "2c92550c-fecb-4d97-aa4c-40e21a34a9f0" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T14:54:31.999Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T14:54:31.899Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 385698 }, "sessions": { "total": 6, "taskScoped": 4, "byGezel": { "radu": 3, "reza": 1, "yong": 2 }, "perTask": { "regional-sales-rollup/1": 2, "regional-sales-rollup/2": 2 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6666666666666666, "maxMessages": 5, "maxContextFill": 0.455108642578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.455108642578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 2, "tasks": 4, "tasksWithReferences": 2, "items": 10, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/data-dictionary", "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/data-dictionary", "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbook/scrape-to-structured" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "authored one parameterized craftbook, ran it twice with different region inputs, and both region summaries exist with distinct contents", "durationMs": 1148460, "timeoutMs": 13516896, "budgetUsedFraction": 0.08496477297746464 }, "timing": { "startedAt": "2026-11-06T14:54:26.200Z", "finishedAt": "2026-11-06T15:13:34.660Z", "timeToFirstArtifactMs": 697837, "timeToLastArtifactWriteMs": 1144111, "timeToFirstTokenMs": 47907, "firstTurnTtftMs": 37163, "timeToFirstToolCallMs": 56497 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 37, "byTool": { "craftbook_read": 2, "list_dir": 7, "list_craftbooks": 1, "craftbook_write": 1, "invoke_craftbook": 2, "list_tasks": 3, "read_task_notes": 4, "get_task": 4, "list_artifacts": 1, "read_artifacts": 1, "read_file": 2, "write_artifact": 2, "read_artifact": 2, "write_file": 3, "write_task_note": 1, "advance_task_step": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 7596, "filePath": "craftbook-author-params", "bytes": 0, "score": 4, "scoreMax": 7, "signals": [], "failReason": "no reusable authored craftbook with at least 2 steps exists yet — author the regional sales rollup as a reusable craftbook template, not as ad-hoc chat work" }, { "atMs": 408575, "filePath": "craftbook-author-params", "bytes": 33, "score": 4, "scoreMax": 7, "signals": [], "failReason": "no task has been created from craftbook \"regional-sales-rollup\" yet — start the recipe on this project for the north region" }, { "atMs": 450183, "filePath": "craftbook-author-params", "bytes": 33, "score": 5, "scoreMax": 7, "signals": [], "failReason": "out/north-summary.md is 0 bytes, need ≥ 80" }, { "atMs": 995890, "filePath": "craftbook-author-params", "bytes": 125, "score": 6, "scoreMax": 7, "signals": [], "failReason": "out/south-summary.md is 0 bytes, need ≥ 80" }, { "atMs": 1138197, "filePath": "craftbook-author-params", "bytes": 202, "score": 6, "scoreMax": 7, "signals": [], "failReason": "out/south-summary.md is 77 bytes, need ≥ 80" }, { "atMs": 1144141, "filePath": "craftbook-author-params", "bytes": 216, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-author-params", "bytes": 216, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 1084733, "kind": "structured", "gezel": "reza", "question": "**Roll up north region sales** is finished. - **`out/north-summary.md`** - `tas…", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-06T14:54:26.204Z] [trial] id=craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc scenario=craftbook-author-params model=underdog-saluki-27b-iq2-mix", "[2026-11-06T14:54:26.204Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "[2026-11-06T14:54:26.205Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T14:54:26.204Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-params-fK8yEl", "[2026-11-06T14:54:26.205Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T14:54:26.208Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T14:54:26.212Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T14:54:26.219Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-params-fK8yEl", "[2026-11-06T14:54:26.219Z] [trial] throughput-scaled ceiling: 90m → 225m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T14:54:31.955Z] [trial] daemon spawned pid=50144 port=https://127.0.0.1:64760", "[2026-11-06T14:54:33.231Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T14:54:33.248Z] [trial] meester=nadia", "[2026-11-06T14:54:33.248Z] [trial] running scenario setup hook", "[2026-11-06T14:54:33.576Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T14:54:33.574Z] [trial] scenario setup complete", "[2026-11-06T14:54:33.639Z] [poll] initial digests hard=2154a7b22efb2db1 soft=e1ccdf4206bb5831", "[2026-11-06T14:54:33.924Z] [poll] hard-progress: 2154a7b22efb2db1 → 03c43a868f8b8e96 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=4/0B)", "[2026-11-06T14:55:26.463Z] [poll] hard-progress: 03c43a868f8b8e96 → 851204d107a473c4 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=36,stream=58) sniff=4/0B)", "[2026-11-06T14:55:32.321Z] [poll] hard-progress: 851204d107a473c4 → 7cda1e1930998fd2 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=43,stream=73) sniff=4/0B)", "[2026-11-06T14:55:55.810Z] [poll] hard-progress: 7cda1e1930998fd2 → 03feae8da7a37312 after 23s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=73,stream=150) sniff=4/0B)", "[2026-11-06T15:01:14.899Z] [poll] hard-progress: 03feae8da7a37312 → 3730fcc46448de2d after 319s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=900,stream=3894) sniff=4/33B)", "[2026-11-06T15:01:56.514Z] [poll] hard-progress: 3730fcc46448de2d → 019244f8b18cef81 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=976,stream=4207) sniff=5/33B)", "[2026-11-06T15:02:02.299Z] [poll] hard-progress: 019244f8b18cef81 → 03fc9256709fe7bd after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=983,stream=4231) sniff=5/33B)", "[2026-11-06T15:02:26.124Z] [poll] hard-progress: 03fc9256709fe7bd → bdf40943750b3c45 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=1035,stream=4393) sniff=5/33B)", "[2026-11-06T15:02:38.025Z] [poll] hard-progress: bdf40943750b3c45 → 05d725048e6bb72d after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=12,slot=1059,stream=4466) sniff=5/33B)" ] }, "metrics": { "process": { "sampleCount": 193, "peakRssMb": 15354, "peakCpuPercent": 1208.6, "firstAtMs": 1, "lastAtMs": 1136878 }, "gpu": { "available": true, "sampleCount": 193, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 193, "peakUsedMb": 31150, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 86644, "totalOutputTokens": 6773, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 57534, "todayTokensCached": 53377, "todayTokensOut": 1894, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 57534, "totalTokensCached": 53377, "totalTokensOut": 1894, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3347, "outputTokens": 70, "cost": 0, "durationMs": 90525, "outputTokensPerSec": 8.663258030369365, "at": "2026-11-06T15:13:19.277Z" }, "lastUpdated": "2026-11-06T15:13:19.277Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T15:13:30.768Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1138370, "meanTokensPerSec": 8.51, "genTokensPerSec": 8.51, "genTokens": 6773, "promptTokensPerSec": 361.7, "peakRssMb": 15354, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 57534, "output": 1894 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T14:54:26.200Z", "finishedAt": "2026-11-06T15:13:34.660Z", "timeToFirstArtifactMs": 697837, "timeToLastArtifactWriteMs": 1144111, "timeToFirstTokenMs": 47907, "firstTurnTtftMs": 37163, "timeToFirstToolCallMs": 56497 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 4 }, "steps": { "activated": 6, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "regional-sales-rollup/1", "stepId": "compute", "activatedAt": "2026-11-06T15:01:52.212Z", "completedAt": "2026-11-06T15:06:04.231Z", "ms": 252019, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "908d7d74-7076-41c3-b9fe-77b73aa31d8d" ] }, { "ref": "regional-sales-rollup/1", "stepId": "write-summary", "activatedAt": "2026-11-06T15:06:04.232Z", "completedAt": "2026-11-06T15:12:29.930Z", "ms": 385698, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "0a004f5f-760b-4c1e-a015-6703207f1a48" ] }, { "ref": "regional-sales-rollup/2", "stepId": "compute", "activatedAt": "2026-11-06T15:01:52.607Z", "completedAt": "2026-11-06T15:09:49.283Z", "ms": 476676, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "f374bafb-a42d-4c6d-8965-d6356d668ae7" ] }, { "ref": "regional-sales-rollup/2", "stepId": "write-summary", "activatedAt": "2026-11-06T15:09:49.284Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "2c92550c-fecb-4d97-aa4c-40e21a34a9f0" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T14:54:31.999Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T14:54:31.899Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 385698 }, "sessions": { "total": 6, "taskScoped": 4, "byGezel": { "radu": 3, "reza": 1, "yong": 2 }, "perTask": { "regional-sales-rollup/1": 2, "regional-sales-rollup/2": 2 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6666666666666666, "maxMessages": 5, "maxContextFill": 0.455108642578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.455108642578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-author-params", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "scenarioId": "craftbook-author-params", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-06T15:38:10.003Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/out/north-summary.md", "artifactBytes": 92, "artifactSha256": "c76d5f80a399dd88940764bc1c364f1f83b9f4873ed65d5e6d049976524cc4dd", "axisSource": "evals/src/craftbooks/authoring/author-params.ts", "axes": [], "scoreAxes": {}, "axisJustifications": {}, "meanScore": null, "justification": "No declared axes. Both final summaries and task intermediates exactly match all source totals and top reps, using one parameterized recipe invoked twice. A concurrent write briefly invents south totals before the assigned writer corrects it. North completes; south remains active at evaluator pass. Final arithmetic succeeds, while write ownership and task closure remain weak.", "reviewBasis": "Read both CSVs, both final summaries, both totals artifacts and authored recipe. Independently recomputed totals in integer cents and top-rep ranking. Traced both invocations, all writes and stage receipts, including the incorrect south draft and its correction.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/out/north-summary.md", "bytes": 92, "sha256": "c76d5f80a399dd88940764bc1c364f1f83b9f4873ed65d5e6d049976524cc4dd", "role": "declaredOutput", "status": "reviewedNoDeclaredAxes" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/artifacts/regional-sales-rollup/tasks/1/totals.md", "bytes": 77, "sha256": "8ed22ef4e5cb63198bfa3ad34ab80152b4e29f8d0bba27bd4959e29a0b0d8918", "role": "taskIntermediate", "status": "reviewedNoDeclaredAxes" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/out/south-summary.md", "bytes": 91, "sha256": "1949e46ae064e49ace7860f4cf9ae2814119f64f2f1398b786a3550506ca531e", "role": "declaredOutput", "status": "reviewedNoDeclaredAxes" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/artifacts/regional-sales-rollup/tasks/2/totals.md", "bytes": 76, "sha256": "68973d21cc3360b8f3fdce3f0e75f16c94be2a10379edbf6c5bcdbc7ca893f51", "role": "taskIntermediate", "status": "reviewedNoDeclaredAxes" } ], "health": { "nativeLaunches": 1, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 36062, "maxSessionContextFill": 0.455108642578125, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.455108642578125 }, "nativeFailureLines": [], "sampledPeaks": { "gpuMiB": 10644, "processTreeRssMiB": 15354, "wholeSystemRamMiB": 31150 }, "note": "Hybrid placement, not full GPU residency. Shared GPU memory and paging unmeasured; whole-system RAM includes other apps." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:40.994Z", "reviewId": "084", "cellId": "r2-saluki-cuda-complex-work-craftbook-author-params", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "scenarioId": "craftbook-author-params", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-author-params", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "dc53de25b6658c0463d60f50a4ebc340e0204d326f3901b573abfe75d0742d0f", "evidenceSha256": "4ce6e6f3c1224abc32f2096e50ac59c3143877868b358038c1f71d7f57bc33c6", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/out/north-summary.md", "bytes": 92, "sha256": "c76d5f80a399dd88940764bc1c364f1f83b9f4873ed65d5e6d049976524cc4dd", "shownBytes": 92, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/artifacts/regional-sales-rollup/tasks/1/totals.md", "bytes": 77, "sha256": "8ed22ef4e5cb63198bfa3ad34ab80152b4e29f8d0bba27bd4959e29a0b0d8918", "shownBytes": 77, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/out/south-summary.md", "bytes": 91, "sha256": "1949e46ae064e49ace7860f4cf9ae2814119f64f2f1398b786a3550506ca531e", "shownBytes": 91, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/artifacts/regional-sales-rollup/tasks/2/totals.md", "bytes": 76, "sha256": "68973d21cc3360b8f3fdce3f0e75f16c94be2a10379edbf6c5bcdbc7ca893f51", "shownBytes": 76, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc\\workspace\\regional-sales-rollup\\data\\north.csv", "bytes": 331, "sha256": "1d565713b89e9dc055719ef769bf6ba5bea1ff888d309aa34767a5231a0af9ba", "shownBytes": 331, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc\\workspace\\regional-sales-rollup\\data\\south.csv", "bytes": 335, "sha256": "e68e801e3fc4700a2f4e7625ae8db017447668c1b36b0d19858bc309ac0be370", "shownBytes": 335, "truncated": false } ], "scoreAxes": {}, "justification": "Both summaries match the CSVs. I re-summed north (1298 units, 47847.95, Cato Meijer at 215 units) and south (2270 units, 83990.00, Nadia Roos at 410 units). Per-task totals.md files show a two-step flow with the region as a parameter. Output paths and format are correct.", "limitations": "The craftbook document, its saved parameter definition, the craftbook_read/write/invoke calls and the authoring order are not in the evidence. Reusability, no baked-in region and no hand-written outputs are unverified. \"Top rep\" is assumed to mean top by units. Source-only review.", "meanScore": null, "durationMs": 16389, "costUsd": 0.019648, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3186, "cache_read_input_tokens": 700, "output_tokens": 683, "output_tokens_details": { "thinking_tokens": 359 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3186, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 683, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3186, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3186 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 683, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3186, "webSearchRequests": 0, "costUSD": 0.019648, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 359, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3186, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-params / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 19m 8s; budget used: 8.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 697837`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":216,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.08496477297746464`; `facts.toolUse.totalToolCalls: 37`; `facts.timing.timeToFirstArtifactMs: 697837`; `facts.outcome.durationMs: 1148460` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15354 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31150 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.51 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 86644 + 6773 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 4s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 6 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 386s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.67 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 46% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 2 / 2 of 4 / 10 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 697837`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":216,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.08496477297746464`\n- `facts.toolUse.totalToolCalls: 37`\n- `facts.timing.timeToFirstArtifactMs: 697837`\n- `facts.outcome.durationMs: 1148460`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 697837`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":216,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.08496477297746464`; `facts.toolUse.totalToolCalls: 37`; `facts.timing.timeToFirstArtifactMs: 697837`; `facts.outcome.durationMs: 1148460`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory inline review\n\nFixed **PASS 8.5/10**, 7/7 checks, unchanged. No declared qualitative axes: **notApplicable**, with absent supporting outputs individually marked **noArtifact**. No qualitative scores were invented.\n\nNo declared axes. Both final summaries and task intermediates exactly match all source totals and top reps, using one parameterized recipe invoked twice. A concurrent write briefly invents south totals before the assigned writer corrects it. North completes; south remains active at evaluator pass. Final arithmetic succeeds, while write ownership and task closure remain weak.\n\nRead both CSVs, both final summaries, both totals artifacts and authored recipe. Independently recomputed totals in integer cents and top-rep ranking. Traced both invocations, all writes and stage receipts, including the incorrect south draft and its correction.\n\n## Positives\n\n- One reusable two-step recipe declares region and uses double-brace parameters in source/output paths; two invocations supply north and south without edits (sniff.latest=7/7; toolUse.byTool.craftbook_write=1, invoke_craftbook=2; captured authoring/invocation receipts).\n- Both intermediate and final values are independently correct: north 1,298 units / 47,847.95 / Cato Meijer; south 2,270 / 83,990.00 / Nadia Roos. No currency is invented in either final summary (outcome.success; artifacts; advisory-consistency-audit.comparisons).\n- Task-scoped totals keep region handoffs separate, and three real stages complete without gate rejections. The final summaries have consistent concise formatting (continuity.steps.completed=3; continuity.sessions.taskScoped=4; captured files).\n\n## Negatives\n\n- Radu writes an incorrect south summary (120 units, $18,400, no top rep) at 15:13:19; Yong replaces it with correct values at 15:13:30. This is a transient error with competing writers, not a defect in the final captured summary (toolUse.byTool.write_file=3; recording write receipts).\n- The evaluator passes after two distinct summaries exist while south remains active in write-summary; it does not establish completion of both recipes (continuity.steps.perStep south write-summary completedAt=null; state.tasks; successCheck source).\n- Recipe gates check minimum size and nonempty content, not numerical correctness. The fixed quality score must not substitute for the independent arithmetic check above (sniff.latest=7/7; final task gate definitions).\n- The run takes 19m08s and 37 calls for two ten-row summaries, although useful computation/handoffs do occur (outcome.durationMs=1148460; toolUse.totalToolCalls=37).\n\n## Performance and GPU fit\n\n**19m8s**, **37 tool calls**, **6,773 engine-generated tokens at 8.51 tok/s**, prefill **361.7 tok/s** (outcome.durationMs, toolUse.totalToolCalls, perf.derived). Sampled peak GPU **10644/12227 MiB**, process-tree RSS **15354 MiB**, whole-system RAM **31150/31865 MiB**. Whole-system RAM includes other applications; shared GPU memory and paging are unmeasured. No new campaign peak.\n\nOne native 0.1.57 CUDA launch; **65,536-token context**, one slot, f16 KV, **39/64 dense FFN layers on GPU and 25 on CPU**. Largest released native slot **36062 tokens (55.0%)**; maximum recorded session fill **45.51%**. No compaction. No native endpoint failure, explicit OOM or context overflow found. These observations demonstrate hybrid execution within VRAM, not full GPU residency or absence of unmeasured shared-memory paging.\n\n## Tactical experiments — not applied\n\nIn the authoring guidance for computation workflows, use a structured intermediate schema with exact numeric totals, source references and declared ranking rules; have the writer copy those fields rather than recompute. The captured compute/write prompts already request this separation and correct intermediates exist, so the main next experiment is enforcing it, not adding fixture-specific expected values or lowering temperature without evidence.\n\nFor reports with tabular totals, attach a reusable gate in packages/script-stdlib/scripts that recomputes integer-cents aggregates from the supplied source path and compares the published fields. Configure the field mapping/ranking as inputs instead of embedding north/south or known totals. Require final summary fields to agree with the current intermediate artifact hash. The transient wrong draft and size-only gates justify this experiment; leave this run's gates unchanged.\n\n## Strategic experiments — not applied\n\nIn packages/service/src/tasks/step-runtime.ts and writes-off-deliverables.ts, scope writes to the active step owner and cancel or reject late writes from a completed predecessor when a successor owns the same output. Trace the Radu/Yong collision before choosing the exact concurrency mechanism; the evidence proves the collision but does not alone prove its scheduler cause.\n\nIn a future separate revision of evals/src/craftbooks/authoring/author-params.ts, report numerical correctness and both task terminal states as distinct checks. The current structural pass is legitimate under its frozen definition, but alone cannot show either property. Preserve historical scores and create an explicit evaluator version boundary for such a change.\n\n## Generalization and next experiment\n\nR1 and R2 both pass fixed 8.5 with correct totals, and both stop with south still active. R2 final formatting is consistent and omits the unsupported dollar sign seen in R1's north report, but adds a transient incorrect south write. Do not claim improved reliability from the identical composite or one corrected final artifact.\n\nThe immediately preceding export-generalize timeout uses single-brace paths and mixed filtering policies; this run shows that the same unchanged host/model can use proper parameter interpolation and produce accurate distinct outputs. Unrelated route-multi completes before the requested-path rewrite, and invoice-run completes while its ledger stays unchanged. Together these motivate generic write ownership, content-bound validation and separate workflow-completion checks. Test across data, communications and invoicing; measure stale writes, correct final values, task closure and elapsed time rather than adding scenario hints.\n\n## Strategic note\n\nThis trial demonstrates real parameter reuse and correct numerical work. It also exposes an erroneous concurrent write and the evaluator's early-stop boundary. Final artifact correctness, recipe closure and native stability are separate findings. No source, gate, prompt, tuning or retry was changed.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/facts.json): pass, fixed 8.5, 34.48 min, 8.5 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/facts.json): fail, fixed 4.6, 7.47 min, 114.45 tok/s.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-36-19-588Z-swlr/facts.json): fail, fixed 5, 7.95 min, 110.25 tok/s.\n\nOnly same-campaign repetitions share the frozen comparison identity. Other-model history is contextual, not a controlled native-version comparison.\n\n## Raw evidence\n\nfacts.json, score.json, state.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json, recording/chat-events.jsonl and captured workspace/artifacts.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-author-gate-script", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-author-gate-script", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 2, "runId": "2026-11-03-windows-saluki-native-0-1-57-original", "segment": "original", "disposition": "runner-terminal", "eligible": false, "success": false, "result": { "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "scenarioId": "craftbook-author-gate-script", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-06T15:16:17.979Z", "finishedAt": "2026-11-06T18:45:16.888Z", "durationMs": 12538909, "success": false, "reason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count.. Last failure: task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count.", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-gate-script\\craftbook-author-gate-script\\craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "failureMode": "model-stuck", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-author-gate-script", "score": 4, "bytes": 5225, "deliverableMissing": false, "milestones": 0, "failReason": "task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, "failureClass": "infra", "failureClassRule": "sandbox-unavailable", "failureClassEvidence": "denyNet requires an enforceable OS network boundary. This platform has no supported boundary, so the script was not started. logsTail=\"[stde" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "scenarioId": "craftbook-author-gate-script", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "infra", "includedInModelAggregate": false, "failureClassRule": "sandbox-unavailable", "evidence": [ { "source": "result", "path": "failureClass", "value": "infra" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count.. Last failure: task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 3422125 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 5225, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } } ] }, "quality": { "score": 8, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 80% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 3422125 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-gate-script", "bytes": 5225, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.695735304170425 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 134 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 3422125 }, { "source": "facts", "path": "outcome.durationMs", "value": 12538909 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.2, "band": "needs-tuning" }, "facts": { "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "scenarioId": "craftbook-author-gate-script", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-gate-script\\craftbook-author-gate-script\\craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 2109, "peakRssMb": 14348, "peakCpuPercent": 1399.5, "firstAtMs": 1, "lastAtMs": 12519534 }, "gpu": { "available": true, "sampleCount": 2109, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 2109, "peakUsedMb": 30798, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 469228, "totalOutputTokens": 87649, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 146064, "todayTokensCached": 117845, "todayTokensOut": 10577, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 146064, "totalTokensCached": 117845, "totalTokensOut": 10577, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 10 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 18532, "cachedInputTokens": 17218, "outputTokens": 1292, "cost": 0, "durationMs": 171197, "outputTokensPerSec": 8.425704936995144, "at": "2026-11-06T18:45:02.474Z" }, "lastUpdated": "2026-11-06T18:45:02.474Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T18:45:05.284Z" } }, "source": "engine-log" }, "derived": { "durationMs": 12521176, "meanTokensPerSec": 8.24, "genTokensPerSec": 8.24, "genTokens": 87649, "promptTokensPerSec": 312.84, "peakRssMb": 14348, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 146064, "output": 10577 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 4 }, "steps": { "activated": 4, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 4, "redrives": 0, "perStep": [ { "ref": "inventory-health-check/1", "stepId": "build", "activatedAt": "2026-11-06T16:08:17.508Z", "completedAt": null, "ms": null, "gateRejections": 2, "toolCalls": 7, "sessionIds": [ "37c88de5-8da5-49f8-aff1-be7f482a9257" ] }, { "ref": "inventory-health-check/2", "stepId": "build", "activatedAt": "2026-11-06T17:45:34.688Z", "completedAt": null, "ms": null, "gateRejections": 2, "toolCalls": 17, "sessionIds": [ "04b4e6b7-c7d0-4498-adce-9bdfd0f25826" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T15:16:23.855Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T15:16:23.761Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 4, "taskScoped": 2, "byGezel": { "anniina": 2, "patrick": 1, "reza": 1 }, "perTask": { "inventory-health-check/1": 1, "inventory-health-check/2": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.5, "maxMessages": 10, "maxContextFill": 0.5998077392578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5998077392578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 2, "tasks": 4, "tasksWithReferences": 2, "items": 10, "citations": [ "knowledge://bendyline/handboek/project-type/inventory-reorder", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/craftbook/data-export-migrate", "knowledge://bendyline/handboek/craftbook/web-app-crud", "knowledge://bendyline/handboek/craftbook/playtest-report", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/data-quality-audit", "knowledge://bendyline/handboek/project-type/inventory-reorder", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/craftbook/data-export-migrate" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count.. Last failure: task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count.", "durationMs": 12538909, "timeoutMs": 18022528, "budgetUsedFraction": 0.695735304170425 }, "timing": { "startedAt": "2026-11-06T15:16:17.979Z", "finishedAt": "2026-11-06T18:45:16.888Z", "timeToFirstArtifactMs": 3422125, "timeToLastArtifactWriteMs": 12524496, "timeToFirstTokenMs": 46296, "firstTurnTtftMs": 35728, "timeToFirstToolCallMs": 57989 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 134, "byTool": { "craftbook_read": 10, "read_file": 25, "list_dir": 9, "list_craftbooks": 3, "how_do_i": 16, "search": 10, "search_documents": 1, "read_document": 10, "craftbook_write": 8, "invoke_craftbook": 2, "get_task": 7, "read_task_notes": 5, "stat": 3, "message_gezel": 1, "write_task_note": 3, "set_task_status": 1, "write_file": 13, "advance_task_step": 3, "replace_in_file": 2, "run_installed_script": 1, "list_packages": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 7746, "filePath": "craftbook-author-gate-script", "bytes": 0, "score": 3, "scoreMax": 5, "signals": [], "failReason": "no reusable authored craftbook with at least 2 steps exists yet — author the inventory-report craftbook as a reusable template" }, { "atMs": 3084745, "filePath": "craftbook-author-gate-script", "bytes": 23, "score": 2, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-report\" step \"build\" references gate script \"checkInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" }, { "atMs": 3120250, "filePath": "craftbook-author-gate-script", "bytes": 23, "score": 2, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-report\" step \"build\" references gate script \"checkInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" }, { "atMs": 3423126, "filePath": "craftbook-author-gate-script", "bytes": 634, "score": 3, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-report\" step \"build\" references gate script \"checkInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" }, { "atMs": 3559492, "filePath": "craftbook-author-gate-script", "bytes": 634, "score": 3, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-report\" step \"build\" references gate script \"checkInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" }, { "atMs": 3565359, "filePath": "craftbook-author-gate-script", "bytes": 634, "score": 3, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-report\" step \"build\" references gate script \"checkInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" }, { "atMs": 3654230, "filePath": "craftbook-author-gate-script", "bytes": 1283, "score": 3, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-report\" step \"build\" references gate script \"checkInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" }, { "atMs": 4674910, "filePath": "craftbook-author-gate-script", "bytes": 1281, "score": 3, "scoreMax": 5, "signals": [], "failReason": "craftbook \"inventory-health-report\" step \"build\" references gate script \"checkInventoryReport\" but the book does not carry that script source — embed the script in the craftbook itself" }, { "atMs": 8915733, "filePath": "craftbook-author-gate-script", "bytes": 5073, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/1 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 8957725, "filePath": "craftbook-author-gate-script", "bytes": 5073, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 9508738, "filePath": "craftbook-author-gate-script", "bytes": 5241, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 11081087, "filePath": "craftbook-author-gate-script", "bytes": 5225, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 11199927, "filePath": "craftbook-author-gate-script", "bytes": 5225, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } ], "latest": { "filePath": "craftbook-author-gate-script", "bytes": 5225, "score": 4, "scoreMax": 5, "signals": [], "failReason": "task inventory-health-check/2 (from craftbook \"inventory-health-report\") has status \"paused\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-06T15:16:17.982Z] [trial] id=craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg scenario=craftbook-author-gate-script model=underdog-saluki-27b-iq2-mix", "[2026-11-06T15:16:17.982Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-06T15:16:17.982Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-gate-script\\craftbook-author-gate-script\\craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "[2026-11-06T15:16:17.982Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-06T15:16:17.982Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-gate-script-mZzFKo", "[2026-11-06T15:16:17.985Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-06T15:16:17.989Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-06T15:16:17.994Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-gate-script-mZzFKo", "[2026-11-06T15:16:17.994Z] [trial] throughput-scaled ceiling: 120m → 300m at 7.99 tok/s (reference 20 tok/s)", "[2026-11-06T15:16:23.814Z] [trial] daemon spawned pid=18560 port=https://127.0.0.1:56687", "[2026-11-06T15:16:25.136Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-06T15:16:25.152Z] [trial] running scenario setup hook", "[2026-11-06T15:16:25.152Z] [trial] meester=aldric", "[2026-11-06T15:16:25.499Z] [trial] scenario setup complete", "[2026-11-06T15:16:25.501Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-06T15:16:25.569Z] [poll] initial digests hard=5e1e17c75a9e565e soft=ba6ecbec8157b4a8", "[2026-11-06T15:16:25.865Z] [poll] hard-progress: 5e1e17c75a9e565e → d7e8d39c94b77a59 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=3/0B)", "[2026-11-06T15:17:18.222Z] [poll] hard-progress: d7e8d39c94b77a59 → d330ee8ff806c9d1 after 52s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=37,stream=64) sniff=3/0B)", "[2026-11-06T15:17:30.061Z] [poll] hard-progress: d330ee8ff806c9d1 → e63bd7d3797e5f09 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=60,stream=129) sniff=3/0B)", "[2026-11-06T15:17:35.866Z] [poll] hard-progress: e63bd7d3797e5f09 → 3e52df2c6f77c4c9 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=63,stream=132) sniff=3/0B)", "[2026-11-06T15:17:59.620Z] [poll] hard-progress: 3e52df2c6f77c4c9 → 80e35b30557ee461 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=100,stream=235) sniff=3/0B)", "[2026-11-06T15:19:04.520Z] [poll] hard-progress: 80e35b30557ee461 → 57007fdc8a58dc69 after 65s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=236,stream=658) sniff=3/0B)", "[2026-11-06T15:19:34.009Z] [poll] hard-progress: 57007fdc8a58dc69 → 392486547d91ae9b after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=295,stream=838) sniff=3/0B)", "[2026-11-06T15:20:03.327Z] [poll] hard-progress: 392486547d91ae9b → 4ad2768f60a64c0e after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=331,stream=952) sniff=3/0B)", "[2026-11-06T15:20:38.734Z] [poll] hard-progress: 4ad2768f60a64c0e → 36c6f1c1a5564fe0 after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=375,stream=1121) sniff=3/0B)" ] }, "metrics": { "process": { "sampleCount": 2109, "peakRssMb": 14348, "peakCpuPercent": 1399.5, "firstAtMs": 1, "lastAtMs": 12519534 }, "gpu": { "available": true, "sampleCount": 2109, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 2109, "peakUsedMb": 30798, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 469228, "totalOutputTokens": 87649, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 10, "todayTokensIn": 146064, "todayTokensCached": 117845, "todayTokensOut": 10577, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 146064, "totalTokensCached": 117845, "totalTokensOut": 10577, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 10 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 18532, "cachedInputTokens": 17218, "outputTokens": 1292, "cost": 0, "durationMs": 171197, "outputTokensPerSec": 8.425704936995144, "at": "2026-11-06T18:45:02.474Z" }, "lastUpdated": "2026-11-06T18:45:02.474Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-06T18:45:05.284Z" } }, "source": "engine-log" }, "derived": { "durationMs": 12521176, "meanTokensPerSec": 8.24, "genTokensPerSec": 8.24, "genTokens": 87649, "promptTokensPerSec": 312.84, "peakRssMb": 14348, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 146064, "output": 10577 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-06T15:16:17.979Z", "finishedAt": "2026-11-06T18:45:16.888Z", "timeToFirstArtifactMs": 3422125, "timeToLastArtifactWriteMs": 12524496, "timeToFirstTokenMs": 46296, "firstTurnTtftMs": 35728, "timeToFirstToolCallMs": 57989 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 4 }, "steps": { "activated": 4, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 4, "redrives": 0, "perStep": [ { "ref": "inventory-health-check/1", "stepId": "build", "activatedAt": "2026-11-06T16:08:17.508Z", "completedAt": null, "ms": null, "gateRejections": 2, "toolCalls": 7, "sessionIds": [ "37c88de5-8da5-49f8-aff1-be7f482a9257" ] }, { "ref": "inventory-health-check/2", "stepId": "build", "activatedAt": "2026-11-06T17:45:34.688Z", "completedAt": null, "ms": null, "gateRejections": 2, "toolCalls": 17, "sessionIds": [ "04b4e6b7-c7d0-4498-adce-9bdfd0f25826" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-06T15:16:23.855Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-06T15:16:23.761Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 4, "taskScoped": 2, "byGezel": { "anniina": 2, "patrick": 1, "reza": 1 }, "perTask": { "inventory-health-check/1": 1, "inventory-health-check/2": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.5, "maxMessages": 10, "maxContextFill": 0.5998077392578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5998077392578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "scenarioId": "craftbook-author-gate-script", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-06T19:42:26.654Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/out/inventory-report.json", "artifactBytes": 1426, "artifactSha256": "c5c121f07321c7967dfc8a3f1b4c057c2e247d4aee5cce31e537b521e3caca05", "axisSource": "evals/src/craftbooks/authoring/gate-script.ts", "axes": [], "scoreAxes": {}, "axisJustifications": {}, "meanScore": null, "justification": "No declared axes. Final report correctly covers all ten items and totals 583 units/7385.60. Neither task completes a stage or runs its gate successfully. The embedded script has SDK defects; two attempts were blocked by a missing sandbox launcher. Restoring that dependency did not produce a later gate attempt. Fixed 5.2 is excluded as infrastructure failure.", "reviewBasis": "Read final report, fixture, helper and workspace draft; matched all three output hashes to the last full review; recomputed integer-cent totals and exact item coverage; compared final task script to the reviewed installed version. Examined all 134 tool calls, terminal state, native logs, four gate rejections and successful sandbox isolation probe. No generated code executed by reviewer.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/out/inventory-report.json", "bytes": 1426, "sha256": "c5c121f07321c7967dfc8a3f1b4c057c2e247d4aee5cce31e537b521e3caca05", "role": "declaredOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/checkInventoryReport.mjs", "bytes": 3730, "sha256": "b4cdacb4ec2d1a72290eafa02b4586635175fc395d7a245a93439324d79aaf66", "role": "supportingOutput", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/craftbooks/inventory-health-report.json", "bytes": 4368, "sha256": "50210a084f28f4617f24c3b2a389381adeb88169b9f58a3cb764f217114ef536", "role": "workspaceDraft", "status": "reviewedNoDeclaredAxes", "matchesPreviousReviewedSnapshot": true }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/out/inventory-report.json.verified.json", "role": "oldRecipeVerificationOutput", "status": "noArtifact", "bytes": 0, "note": "Old version 1.0.0 expected a verification file; current task version 1.0.1 requires a verification note. Neither verification stage completed." } ], "health": { "nativeLaunches": 2, "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 46670, "maxSessionContextFill": 0.5998077392578125, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5998077392578125 }, "nativeFailureLines": [ "[llama-cpp] /v1/chat/completions unreachable at http://127.0.0.1:56691: fetch failed", "2026-11-06T15:20:54.489Z DEBUG [chat] runSend#c322cac0 EXIT throw afterMs=268999 [llama-cpp] /v1/chat/completions unreachable at http://127.0.0.1:56691: fetch failed", "2026-11-06T15:20:55.789Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-38712-1793978455789\",\"pid\":38712,\"code\":3221226505,\"signal\":null,\"expected\":true,\"expectedReason\":\"memory-pressure\",\"uptimeMs\":265980,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "sampledPeaks": { "gpuMiB": 10648, "processTreeRssMiB": 14348, "wholeSystemRamMiB": 30798 }, "sandboxRestoredAt": "2026-11-06T18:37:17Z", "sandboxProbe": [ "2026-11-06T18:37:36.614Z INFO [sandbox] [sandbox] windows-mxc available (tier=base-container, 450ms probe)" ], "note": "Hybrid placement; shared GPU memory and paging unmeasured. Whole-system RAM includes other applications. One native endpoint-loss/nonzero-exit episode recovered within the trial; cleanup memory-pressure label does not establish OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:55.748Z", "reviewId": "085", "cellId": "r2-saluki-cuda-complex-work-craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "scenarioId": "craftbook-author-gate-script", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-author-gate-script", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03", "promptSha256": "0ddc18591e102a582450701b60b0b0780269b2fb2edbe2d9ca780e754a3b64f6", "evidenceSha256": "3d9fd32345ce887be4c9ff3e2bc5725dc9ec08bfd8426ef7be3a19442f7123a9", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/out/inventory-report.json", "bytes": 1426, "sha256": "c5c121f07321c7967dfc8a3f1b4c057c2e247d4aee5cce31e537b521e3caca05", "shownBytes": 1426, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/checkInventoryReport.mjs", "bytes": 3730, "sha256": "b4cdacb4ec2d1a72290eafa02b4586635175fc395d7a245a93439324d79aaf66", "shownBytes": 3730, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/craftbooks/inventory-health-report.json", "bytes": 4368, "sha256": "50210a084f28f4617f24c3b2a389381adeb88169b9f58a3cb764f217114ef536", "shownBytes": 4368, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\r2-saluki-cuda-complex-work-craftbook-author-gate-script\\craftbook-author-gate-script\\craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg\\workspace\\inventory-health-check\\data\\inventory.json", "bytes": 1006, "sha256": "2a6f44cb0bb8d193c2abf527264e828fbf882e8a744180a786e784fe0688b439", "shownBytes": 1006, "truncated": false } ], "scoreAxes": {}, "justification": "Report matches inventory (10 SKUs, 583 units, correct buckets). Embedded gate imports from the SDK, parses and cross-checks the report, and fails otherwise. Gaps: the craftbook has only the build step, with no verification step after it. The .verified.json file is missing.", "limitations": "Source-only review. No execution evidence shows the craftbook was invoked or the gate ran. The craftbook's saved shape and the missing verified file suggest the run was incomplete. The report's generatedAt and its build mechanism are unverified.", "meanScore": null, "durationMs": 17777, "costUsd": 0.034867999999999996, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 7956, "cache_read_input_tokens": 700, "output_tokens": 297, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 7956, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 297, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 7956, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 7956 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 297, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 7956, "webSearchRequests": 0, "costUSD": 0.034867999999999996, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 7956, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-gate-script / underdog-saluki-27b-iq2-mix\n\n**Composite: 5.2 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 208m 59s; budget used: 69.6%)\n**Model aggregate eligibility: excluded from model aggregate (infra / sandbox-unavailable).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: tas...`; `facts.timing.timeToFirstArtifactMs: 3422125`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":5225,\"score\":4,\"scoreMax\":5,\"signals\":[],\"failReason\":\"task inventory-health-check/2 (from craftbook \\\"inventory-health-report...` |\n| Output quality (25%) | 8 / 10 | The latest scenario gate reached 80% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 3422125`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":5225,\"score\":4,\"scoreMax\":5,\"signals\":[],\"failReason\":\"task inventory-health-check/2 (from craftbook \\\"inventory-health-report...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.695735304170425`; `facts.toolUse.totalToolCalls: 134`; `facts.timing.timeToFirstArtifactMs: 3422125`; `facts.outcome.durationMs: 12538909` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 8 + 0.2 * 2.5 + 0.15 * 10 = **5.2**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14348 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10648 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30798 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.24 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 469228 + 87649 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 4s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 4 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 4 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 2 / 0.50 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 1 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 60% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 2 / 2 of 4 / 10 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: tas...`\n- `facts.timing.timeToFirstArtifactMs: 3422125`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":5225,\"score\":4,\"scoreMax\":5,\"signals\":[],\"failReason\":\"task inventory-health-check/2 (from craftbook \\\"inventory-health-report...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 80% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 3422125`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":5225,\"score\":4,\"scoreMax\":5,\"signals\":[],\"failReason\":\"task inventory-health-check/2 (from craftbook \\\"inventory-health-report...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.695735304170425`\n- `facts.toolUse.totalToolCalls: 134`\n- `facts.timing.timeToFirstArtifactMs: 3422125`\n- `facts.outcome.durationMs: 12538909`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on craftbook: inventory report gate script; missing signals stayed unchanged: tas...`; `facts.timing.timeToFirstArtifactMs: 3422125`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":5225,\"score\":4,\"scoreMax\":5,\"signals\":[],\"failReason\":\"task inventory-health-check/2 (from craftbook \\\"inventory-health-report...`\n- Quality: The latest scenario gate reached 80% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 3422125`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-gate-script\",\"bytes\":5225,\"score\":4,\"scoreMax\":5,\"signals\":[],\"failReason\":\"task inventory-health-check/2 (from craftbook \\\"inventory-health-report...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.695735304170425`; `facts.toolUse.totalToolCalls: 134`; `facts.timing.timeToFirstArtifactMs: 3422125`; `facts.outcome.durationMs: 12538909`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Advisory artifact review\n\nFixed **FAIL 5.2/10**, **excluded from model aggregate: infra / sandbox-unavailable**. The standard runner halted the campaign at **85/129**. The fixed result and classification remain unchanged. No declared qualitative axes: **notApplicable**; the absent old verification output is **noArtifact**. No advisory scores invented.\n\nNo declared axes. Final report correctly covers all ten items and totals 583 units/7385.60. Neither task completes a stage or runs its gate successfully. The embedded script has SDK defects; two attempts were blocked by a missing sandbox launcher. Restoring that dependency did not produce a later gate attempt. Fixed 5.2 is excluded as infrastructure failure.\n\nRead final report, fixture, helper and workspace draft; matched all three output hashes to the last full review; recomputed integer-cent totals and exact item coverage; compared final task script to the reviewed installed version. Examined all 134 tool calls, terminal state, native logs, four gate rejections and successful sandbox isolation probe. No generated code executed by reviewer.\n\n## Positives\n\n- Final report faithfully covers all ten SKUs with correct names/quantities and correct totals of 583 units and 7385.60, including negative stock; buckets match the authored <20 policy (sniff.latest=4/5, artifacts.otherFileCount=5; independent consistency audit).\n- The author eventually saves a substantive embedded script map and invokes a two-step recipe; this is real authoring progress beyond the earlier missing script source (toolUse.byTool.craftbook_write=8, invoke_craftbook=2; state.tasks task 2 version 1.0.1).\n- After one native endpoint recovery, inference continues through 87,649 generated tokens and one successful mid-loop compaction (perf.derived.genTokens; continuity.compaction.midTurn=1, compactFailed=0; native log).\n\n## Negatives\n\n- Both authored tasks remain paused at build; no stage or gate approval completes after 208m59s and 134 calls (outcome.failureMode=model-stuck; continuity.steps.completed=0, gateApprovals=0, gateRejections=4; toolUse.totalToolCalls).\n- Two initial gate rejections lack embedded source and two later ones lack the enforceable OS boundary. The latter infrastructure classification excludes this result from model aggregates even though authoring defects also exist (continuity.steps.gateRejections=4; result.failureClassRule=sandbox-unavailable; score.eligibility).\n- Carried script imports gezel from the checks-only SDK entry and passes an object to gateResult instead of (boolean, detail). These are static defects, not observed post-repair execution failures: no later gate runs (sniff.latest=4/5; state.tasks[task2].craftbook.scripts; packages/sdk/src/checks.ts).\n- A new recipe version and repeated workspace draft writes do not repair the already instantiated task snapshot. Four completed feedback attempts exhaust while the missing terminal-state signal stays unchanged (outcome.reason; toolUse.byTool.write_file=13, craftbook_write=8; final three tool receipts).\n\n## Infrastructure and chronology\n\nThe original 1.0.0 recipe placed source inside gate.scripts entries rather than the top-level scripts map. Build could not find the script. Its report also had changing arithmetic/schema defects; all overwritten drafts remain in the four advisory-progress snapshots. At 17:44 the author saved 1.0.1 with an actual scripts map and invoked a second task at 17:45. The final report, rewritten at 17:54, is correct under that new contract. The 17:57 and 18:22 gate attempts stopped before script execution because denyNet had no enforceable boundary. Version 1.0.2 removes only meta.kind=gate and does not update task 2's 1.0.1 snapshot.\n\nAt 18:37:17 the existing pinned MXC 1.0.0 executable and original verification sentinel were copied from the read-only dev cache into only this disposable trial home. SHA-256 and Microsoft signature matched. The daemon isolation probe passed at 18:37:36; authenticated TLS health confirmed active windows-mxc/base-container at 18:38:03. The last three calls inspect task 2, read the workspace draft and rewrite identical bytes. Neither gate nor verification runs afterward. Repair-exhausted ends the trial at 18:45:16.888Z; the parent exits 18:47:10 with code 1.\n\nNo source, gate, tuning, security policy, trust, GEZEL environment variable, task state or campaign journal was changed by the reviewer. No manual resume, extra model message or capability retry occurred. Standard post-trial identity verification reports no comparisonIssue. The restoration fixes an environment prerequisite; it does not establish that the authored script works. The facts extractor's autoAnswer.total=0 is retained; task pause auto-retries appear separately in logs and are not silently added to that score field.\n\n## Performance and fit\n\n**208m59s**, **134 calls**, **87,649 engine-generated tokens at 8.24 tok/s**, prefill **312.84 tok/s**. Peak sampled GPU **10,648/12,227 MiB**, process-tree RSS **14,348 MiB**, system RAM **30,798/31,865 MiB** (facts.outcome, toolUse, perf). Whole-system RAM includes other applications; paging/shared GPU memory were not measured.\n\nTwo native launches, **65,536-token allocated context**, one slot, f16 KV, **39/64 dense FFN layers on GPU and 25 on CPU**. Maximum released native slot **46670 tokens**; recorded session fill **59.98%**; one successful compaction. At 15:20 an endpoint loss was followed by native exit 3221226505; replacement served at 15:21. This is the fifth campaign episode, not a new episode after the 16:28 audit. The cleanup label memory-pressure is not proof of OOM. No explicit CUDA OOM or context overflow found.\n\n## Tactical experiments — not applied\n\nIn authoring guidance associated with packages/mcp/src/installed-script-miss.ts and the script authoring documentation, return one canonical minimal SDK example using gezel from the main entry and gateResult(check.ok, check.detail) from /checks. Bound recovery guidance to the exact rejected field and current task snapshot version instead of another broad documentation search. Evidence: 16 how_do_i calls, ten search calls, eight craftbook_write calls and unchanged SDK mistakes in the final embedded source. Test on unrelated CSV validation and document verification with no scenario-specific filenames or expected totals.\n\nIn a separately versioned harness environment, configure GEZEL_MXC_EXEC to the existing verified launcher before each disposable Windows daemon starts; require its isolation probe before admitting script-dependent trials. This exact missing dependency recurred in R1 and R2. Preserve denyNet and all trust policies. Measure fresh-home preflight failures and successful containment across inventory and an unrelated scripted computation. This is an environment experiment, not a sampling recommendation or a retry of this failed attempt.\n\n## Strategic experiments — not applied\n\nIn packages/service/src/tasks/task-creation.ts and step-runtime.ts, surface installed-recipe and instantiated-task versions together and require an explicit supported update path before treating a catalog save as task repair. In packages/mcp/src/installed-script-miss.ts, link diagnostics to the actual embedded source map and SDK contract. Validate imports and result shapes before invocation where feasible, while retaining the sandbox as the execution boundary.\n\nIn a future separate evaluator version of evals/src/craftbooks/authoring/gate-script.ts, distinguish terminal status, successful custom gate receipt and final verifier receipt. Do not turn the present structural pass definition into a retroactive score change. A valid report and a nontrivial-looking script are insufficient evidence of executed verification. Preserve rejected attempts and classify environment admission independently of task repair feedback.\n\n## Historical comparison and generalization\n\nR1 passes fixed 8.1 after model status completion without successful gate execution, and its final total value is wrong by 10. R2 ends excluded at 5.2 with correct final arithmetic but no completed stage. Both needed the same narrow sandbox dependency restoration. This is neither a controlled performance regression nor evidence that the earlier passing workflow was fully verified. The current infrastructure exclusion must stay outside aggregate capability arithmetic.\n\nAuthor-params R2 correctly computes two regional summaries but stops while one task is still active. Export-generalize R2 produces faithful requested arrays/reports yet never closes its authored recipe; route-multi and invoice-run also separate lifecycle or review claims from final evidence. These diverse cases motivate generic content-bound verification, write ownership and explicit lifecycle receipts, rather than scenario hints or a stronger-model recommendation. Measure both correct final data and real task/gate completion in the next separately authorized experiment.\n\n## Last three same-scenario trials\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/facts.json): pass, fixed 8.1, 160.80 min, 8.31 tok/s; eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/facts.json): fail, fixed 4.5, 43.09 min, 116.21 tok/s; eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-02-26-265Z-bv1f/facts.json): fail, fixed 5.2, 5.19 min, 101.74 tok/s; excluded infra.\n\nOnly same-campaign repetitions share the frozen identity; other-model history has different native/build/model conditions and is contextual only.\n\n## Strategic note\n\nThe recurring missing fresh-home sandbox is an infrastructure problem. Independent static authoring defects and repeated ineffective repairs remain useful task-handling evidence, but this excluded trial must not be relabeled a capability failure or rerun to obtain a pass. The standard campaign runner refuses continuation once stopped (evals/src/scorecard-campaign/run.ts:367). A new baseline is required to attempt only the 44 still-unattempted entries.\n\n## Raw evidence\n\nfacts.json, score.json, result.json, state.json, advisory-artifact-review.json, advisory-consistency-audit.json, advisory-tool-evidence.json, historical-comparison.json, recording/chat-events.jsonl, daemon.log, four preserved advisory-progress snapshots, and root mxc-home-repair-r2.json / mxc-home-repair-r2-health.json.\n" }, { "cellId": "r3-saluki-cuda-core-tictactoe", "baselineCellId": "r1-saluki-cuda-core-tictactoe", "suite": "core", "scenario": "tictactoe", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T20:16:26.588Z", "finishedAt": "2026-11-09T20:22:03.274Z", "durationMs": 336686, "success": true, "reason": "tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 328413 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 4149, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": 2, "runtimeFailed": 0 } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 4149 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.030909645247726734 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 328413 }, { "source": "facts", "path": "outcome.durationMs", "value": 336686 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "scenarioId": "tictactoe", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 55, "peakRssMb": 10040, "peakCpuPercent": 1240.2, "firstAtMs": 0, "lastAtMs": 321641 }, "gpu": { "available": true, "sampleCount": 55, "peakUtilPercent": 97, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 55, "peakUsedMb": 31367, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12817, "totalOutputTokens": 2468, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 14610, "todayTokensCached": 13784, "todayTokensOut": 581, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 14610, "totalTokensCached": 13784, "totalTokensOut": 581, "totalCost": 0, "medianOutputTokensPerSec": 9.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.3, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7371, "cachedInputTokens": 6892, "outputTokens": 489, "cost": 0, "durationMs": 55223, "outputTokensPerSec": 9.186459219349755, "at": "2026-11-09T20:18:54.649Z" }, "lastUpdated": "2026-11-09T20:18:54.649Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T20:21:58.328Z" } }, "source": "engine-log" }, "derived": { "durationMs": 324963, "meanTokensPerSec": 8.91, "genTokensPerSec": 8.91, "genTokens": 2468, "promptTokensPerSec": 494.62, "peakRssMb": 10040, "peakGpuUtilPercent": 97, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 14610, "output": 581 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tic-tac-toe-game/1", "stepId": "build", "activatedAt": "2026-11-09T20:17:47.874Z", "completedAt": "2026-11-09T20:21:55.812Z", "ms": 247938, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "85f6d1f7-d8c6-436f-b2b2-b32a7bb24da8" ] }, { "ref": "tic-tac-toe-game/1", "stepId": "evaluate", "activatedAt": "2026-11-09T20:21:55.814Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T20:16:33.413Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T20:16:33.320Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 247938 }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "laxmi": 1, "orion": 1 }, "perTask": { "tic-tac-toe-game/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.25, "maxMessages": 4, "maxContextFill": 0.0977935791015625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0977935791015625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "durationMs": 336686, "timeoutMs": 10892587, "budgetUsedFraction": 0.030909645247726734 }, "timing": { "startedAt": "2026-11-09T20:16:26.588Z", "finishedAt": "2026-11-09T20:22:03.274Z", "timeToFirstArtifactMs": 328413, "timeToLastArtifactWriteMs": 328448, "timeToFirstTokenMs": 30446, "firstTurnTtftMs": 19173, "timeToFirstToolCallMs": 81382 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "start_work": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/tic-tac-toe-game/index.html", "finalBytes": 4149, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 329303, "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 4149, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null } ], "latest": { "filePath": "tic-tac-toe-game/workspace/index.html", "bytes": 4149, "score": 6, "scoreMax": null, "signals": [ "name", "grid", "click", "win-detect", "js-parses", "js-size-ok" ], "failReason": null, "runtimePassed": 2, "runtimeFailed": 0 } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T20:16:26.591Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "[2026-11-09T20:16:26.591Z] [trial] id=tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl scenario=tictactoe model=underdog-saluki-27b-iq2-mix", "[2026-11-09T20:16:26.591Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T20:16:26.591Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-tictactoe-sHIrc8", "[2026-11-09T20:16:26.591Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T20:16:26.594Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T20:16:26.598Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T20:16:26.602Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-tictactoe-sHIrc8", "[2026-11-09T20:16:26.602Z] [trial] throughput-scaled ceiling: 120m → 182m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T20:16:33.314Z] [trial] daemon spawned pid=2632 port=https://127.0.0.1:54514", "[2026-11-09T20:16:34.298Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T20:16:34.317Z] [trial] meester=orion", "[2026-11-09T20:16:34.480Z] [trial] sent prompt to meester (621 chars)", "[2026-11-09T20:16:34.535Z] [poll] initial digests hard=d568d6d4f3c01b1f soft=17c51c0f7d19cf8a", "[2026-11-09T20:17:51.866Z] [poll] hard-progress: d568d6d4f3c01b1f → 83f4d1d4873e53e2 after 77s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=164,stream=603) sniff=none)", "[2026-11-09T20:18:51.614Z] [poll] hard-progress: 83f4d1d4873e53e2 → 67d55cce44eebf6d after 60s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=1,slot=345,stream=1133) sniff=none)", "[2026-11-09T20:21:57.942Z] [poll] terminal: success=true reason=tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))", "[2026-11-09T20:21:58.358Z] [capture] dumped 2 session(s)", "[2026-11-09T20:21:58.361Z] [capture] snapshotted telemetry for 2 session(s)", "[2026-11-09T20:21:58.409Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-09T20:22:03.273Z] [trial] daemon shut down", "[2026-11-09T20:22:03.282Z] [trial] PASS duration=336686ms reason=tic-tac-toe-game/workspace/index.html passed sniff (signals: name, grid, click, win-detect, js-parses, js-size-ok) + runtime (2 assertion(s))" ] }, "metrics": { "process": { "sampleCount": 55, "peakRssMb": 10040, "peakCpuPercent": 1240.2, "firstAtMs": 0, "lastAtMs": 321641 }, "gpu": { "available": true, "sampleCount": 55, "peakUtilPercent": 97, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 55, "peakUsedMb": 31367, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12817, "totalOutputTokens": 2468, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 14610, "todayTokensCached": 13784, "todayTokensOut": 581, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 14610, "totalTokensCached": 13784, "totalTokensOut": 581, "totalCost": 0, "medianOutputTokensPerSec": 9.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.3, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7371, "cachedInputTokens": 6892, "outputTokens": 489, "cost": 0, "durationMs": 55223, "outputTokensPerSec": 9.186459219349755, "at": "2026-11-09T20:18:54.649Z" }, "lastUpdated": "2026-11-09T20:18:54.649Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T20:21:58.328Z" } }, "source": "engine-log" }, "derived": { "durationMs": 324963, "meanTokensPerSec": 8.91, "genTokensPerSec": 8.91, "genTokens": 2468, "promptTokensPerSec": 494.62, "peakRssMb": 10040, "peakGpuUtilPercent": 97, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 14610, "output": 581 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T20:16:26.588Z", "finishedAt": "2026-11-09T20:22:03.274Z", "timeToFirstArtifactMs": 328413, "timeToLastArtifactWriteMs": 328448, "timeToFirstTokenMs": 30446, "firstTurnTtftMs": 19173, "timeToFirstToolCallMs": 81382 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tic-tac-toe-game/1", "stepId": "build", "activatedAt": "2026-11-09T20:17:47.874Z", "completedAt": "2026-11-09T20:21:55.812Z", "ms": 247938, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "85f6d1f7-d8c6-436f-b2b2-b32a7bb24da8" ] }, { "ref": "tic-tac-toe-game/1", "stepId": "evaluate", "activatedAt": "2026-11-09T20:21:55.814Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T20:16:33.413Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T20:16:33.320Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 247938 }, "sessions": { "total": 2, "taskScoped": 1, "byGezel": { "laxmi": 1, "orion": 1 }, "perTask": { "tic-tac-toe-game/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.25, "maxMessages": 4, "maxContextFill": 0.0977935791015625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0977935791015625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-tictactoe", "baselineCellId": "r1-saluki-cuda-core-tictactoe", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "scenarioId": "tictactoe", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "scored", "reviewedAt": "2026-11-09T21:51:04.534Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/workspace/tic-tac-toe-game/index.html", "artifactBytes": 4149, "artifactSha256": "ba1e579a210cc799b723fdb598c0028d92b7b7c2e1898f2bac54379efb039468", "axisSource": "evals/src/llm-judge.ts DEFAULT_HTML_AXES", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 6, "functionalCompleteness": 7, "codeQuality": 7, "polish": 4 }, "meanScore": 6, "justification": "Clean board and concise logic. Both players can win; draws, reset, occupied-cell/end-state guards and Enter activation work. The 320px view fits. Cell names never announce X/O and status lacks a live region; presentation remains basic beyond hover and winning-line feedback.", "reviewBasis": "Read complete primary artifacts within the 20KiB cap, all declared companion outputs and relevant scenario contract. Inspected tool receipts and terminal facts. Browser checks/images or production-export probes are saved separately; no artifact or fixed evidence was changed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/workspace/tic-tac-toe-game/index.html", "bytes": 4149, "sha256": "ba1e579a210cc799b723fdb598c0028d92b7b7c2e1898f2bac54379efb039468", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 4149, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T20:16:35.320Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T20:16:39.215Z INFO [chat] [llama-server] launch {\"pid\":26008,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T20:22:02.529Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-26008-1794255722529\",\"pid\":26008,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":323315,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 7859, "peakGpuMiB": 10636, "peakProcessRssMiB": 10040, "peakSystemRamMiB": 31367, "decodeTokensPerSec": 8.91, "prefillTokensPerSec": 494.62, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0977935791015625 }, "note": "Hybrid task placement, 39/64 dense FFN layers on GPU and 25 CPU. Whole-system memory includes other applications; shared GPU memory and paging are unmeasured. Multiple launches alone do not imply a native crash." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:59.499Z", "reviewId": "087", "cellId": "r3-saluki-cuda-core-tictactoe", "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "scenarioId": "tictactoe", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "baselineCellId": "r1-saluki-cuda-core-tictactoe", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "e73eb015692b5ab2d27a3c56569f1d57a08afc27b453ed54075dfaea2cf83392", "evidenceSha256": "ebdb82d12b86e45c96370996f05b8bf791dc5aac1cf9e0d602eef449167bfc5f", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/workspace/tic-tac-toe-game/index.html", "bytes": 4149, "sha256": "ba1e579a210cc799b723fdb598c0028d92b7b7c2e1898f2bac54379efb039468", "shownBytes": 4149, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 6, "functionalCompleteness": 9, "codeQuality": 8, "polish": 5 }, "justification": "The source meets the brief. It has nine static .cell buttons with data-cell 0–8, wired via querySelectorAll. It has turns, win and draw detection, a winner message, and reset. The code is clean and well structured. The styling is tidy but plain: a card-less board, X/O colors, a green win highlight and hover. Extras are limited to aria-labels and the win highlight.", "limitations": "Source-only review; nothing was rendered or run. The file is in workspace/tic-tac-toe-game/, which fits the \"new project\" instruction but not the literal workspace/index.html path. Layout and look are inferred from the CSS.", "meanScore": 7, "durationMs": 18435, "costUsd": 0.020747599999999998, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3757, "cache_read_input_tokens": 856, "output_tokens": 563, "output_tokens_details": { "thinking_tokens": 207 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3757, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 563, "cache_read_input_tokens": 856, "cache_creation_input_tokens": 3757, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3757 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 563, "cacheReadInputTokens": 856, "cacheCreationInputTokens": 3757, "webSearchRequests": 0, "costUSD": 0.020747599999999998, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 207, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3757, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - tictactoe / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 5m 37s; budget used: 3.1%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 328413`; `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":4149,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`; `facts.sniff.latest.bytes: 4149` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.030909645247726734`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 328413`; `facts.outcome.durationMs: 336686` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10040 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 97% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10636 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31367 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.91 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 12817 + 2468 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 4 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 248s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.25 | `continuity.sessions` |\n| Sessions reused across steps | 0 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 10% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 328413`\n- `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":4149,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`\n- `facts.sniff.latest.bytes: 4149`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.030909645247726734`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 328413`\n- `facts.outcome.durationMs: 336686`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 328413`; `facts.sniff.latest: {\"filePath\":\"tic-tac-toe-game/workspace/index.html\",\"bytes\":4149,\"score\":6,\"scoreMax\":null,\"signals\":[\"name\",\"grid\",\"click\",\"win-detect\",\"js-parses\",\"js-size-ok\"],\"failReason\":n...`; `facts.sniff.latest.bytes: 4149`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.030909645247726734`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 328413`; `facts.outcome.durationMs: 336686`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed composite **10/10**, pass; eligibility unchanged. The advisory mean is **6/10**. Clean board and concise logic. Both players can win; draws, reset, occupied-cell/end-state guards and Enter activation work. The 320px view fits. Cell names never announce X/O and status lacks a live region; presentation remains basic beyond hover and winning-line feedback.\n\n| Advisory axis | Score |\n|---|---|\n| visualQuality | 6/10 |\n| functionalCompleteness | 7/10 |\n| codeQuality | 7/10 |\n| polish | 4/10 |\n\n## Positives\n\nThe requested two-player game is delivered in two tool calls, with both standard runtime assertions passing (toolUse.totalToolCalls=2; sniff.latest.runtimePassed=2). Advisory checks additionally confirm X/O wins, draw, reset and blocked occupied/end-state moves.\n\n## Negatives\n\nA fixed 10 does not establish polished accessibility: static aria-labels omit the mark and the status lacks a live region (advisory-ui-checks-20261109/checks.json). Task evaluation was just activated when the trial stopped (continuity.steps.completed=1).\n\n## Performance and fit\n\nDecode 8.91 tok/s; prefill 494.62 tok/s. Peak GPU 10636/12227 MiB, process-tree RSS 10040 MiB, system RAM 31367/31865 MiB (facts.perf). Context 65536, 1 slot, f16; max observed released slot 7859 tokens. Explicit native/OOM/context fault lines: 0. Hybrid placement is not full GPU residency, and shared GPU memory/paging remain unmeasured. Petshop includes image-engine use; a second chat launch alone is not classified as a crash.\n\n## Tactical experiment — not applied\n\nExperiment only: add one generic pre-delivery keyboard/state-announcement check to ../gilde/data/gezel-templates/bu/builder/versions/1.5.1/about.md. Compare interactive forms and unrelated browser games; target the missing accessible state rather than a named scenario.\n\n## Strategic experiment — not applied\n\nExperiment only: extend packages/mcp/src/validate.ts with an optional browser accessibility/state audit and explicit runtime-available status. Preserve fixed gates; measure findings and false positives across forms, dashboards and games.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/facts.json): pass, fixed 8.8, 5.67 min, eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/facts.json): pass, fixed 8.8, 7.99 min, eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/facts.json): pass, fixed 10, 0.82 min, eligible.\n\nUnrelated controls include original R2 codemod sweep (pass 8.5) and constrained communications (fail 6). A successful artifact or compile gate does not establish broad workflow or output quality. The proposals target reusable validation, state boundaries and tool recovery, with no fixture-specific global hint. Original/continuation GEZEL_MXC_EXEC is an explicit comparison boundary; other-model/native history and differing automatic timeout scaling prevent causal speed/quality attribution.\n\n## Recommended next experiment\n\nEvaluate the proposed diagnostic on one unrelated task type and record advisory defects, repair turns and false positives without modifying this campaign's fixed evidence.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, historical-comparison.json, plus advisory-ui-checks-20261109 or advisory production-export checks where present.\n" }, { "cellId": "r3-saluki-cuda-core-petshop", "baselineCellId": "r1-saluki-cuda-core-petshop", "suite": "core", "scenario": "petshop", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T20:25:10.814Z", "finishedAt": "2026-11-09T21:11:32.304Z", "durationMs": 2781490, "success": true, "reason": "pet-shop-website/workspace/index.html passed sniff (signals: pet-vocab, store-vocab, structured-page, working-image, image-asset) + runtime (1 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "requiredRasterMinBytes": 51200 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 487687 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "pet-shop-website/workspace/index.html", "bytes": 7554, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": 1, "runtimeFailed": 0 } }, { "source": "facts", "path": "artifacts.imageFiles", "value": [ { "path": "workspace/pet-shop-website/assets/logo.png", "bytes": 1571902 }, { "path": "artifacts/pet-shop-website/generated/image-2026-11-09T21-11-17-063Z-45739106.png", "bytes": 1571902 }, { "path": "artifacts/pet-shop-website/sessions/2026-11-09_202923_new-session/tool-0-img-0.png", "bytes": 1571902 } ] } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2553562344739592 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 32 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 487687 }, { "source": "facts", "path": "outcome.durationMs", "value": 2781490 } ] }, "behavior": { "score": 0, "ruleId": "B_RED_FLAG_OR_MISSING_ROLE", "summary": "At least one red-flag behavior fired or an expected role was missing.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [ "image-generator" ] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 7, "band": "needs-tuning" }, "facts": { "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "scenarioId": "petshop", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 463, "peakRssMb": 13399, "peakCpuPercent": 1245, "firstAtMs": 0, "lastAtMs": 2760551 }, "gpu": { "available": true, "sampleCount": 463, "peakUtilPercent": 100, "peakMemUsedMb": 10652, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 463, "peakUsedMb": 31506, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 121400, "totalOutputTokens": 18893, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 89665, "todayTokensCached": 66461, "todayTokensOut": 6038, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 89665, "totalTokensCached": 66461, "totalTokensOut": 6038, "totalCost": 0, "medianOutputTokensPerSec": 8.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.2, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9780, "outputTokens": 162, "cost": 0, "durationMs": 73399, "outputTokensPerSec": 8.223899284714866, "at": "2026-11-09T21:09:45.761Z" }, "lastUpdated": "2026-11-09T21:09:45.761Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:11:19.240Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2762961, "meanTokensPerSec": 8.17, "genTokensPerSec": 8.17, "genTokens": 18893, "promptTokensPerSec": 392.99, "peakRssMb": 13399, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 89665, "output": 6038 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "pet-shop-website/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-09T20:26:47.187Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "9a43aeb6-301a-430a-b6a5-350bba964a92" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T20:25:16.432Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T20:25:16.333Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 5, "taskScoped": 1, "byGezel": { "darika": 1, "kumar": 1, "naoki": 1, "vanya": 2 }, "perTask": { "pet-shop-website/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 7, "maxContextFill": 0.18939208984375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.18939208984375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 6 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "pet-shop-website/workspace/index.html passed sniff (signals: pet-vocab, store-vocab, structured-page, working-image, image-asset) + runtime (1 assertion(s))", "durationMs": 2781490, "timeoutMs": 10892587, "budgetUsedFraction": 0.2553562344739592 }, "timing": { "startedAt": "2026-11-09T20:25:10.814Z", "finishedAt": "2026-11-09T21:11:32.304Z", "timeToFirstArtifactMs": 487687, "timeToLastArtifactWriteMs": 2674918, "timeToFirstTokenMs": 28689, "firstTurnTtftMs": 19110, "timeToFirstToolCallMs": 59307 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Builder", "Illustrator" ], "missingExpectedRoles": [ "image-generator" ] }, "toolUse": { "totalToolCalls": 32, "byTool": { "suggest_work": 1, "start_work": 1, "ensure_gezel": 3, "message_gezel": 3, "find_files": 1, "write_file": 13, "validate": 3, "list_dir": 1, "read_file": 2, "replace_in_file": 1, "generate_image": 1, "delegate_image_generator": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/pet-shop-website/index.html", "finalBytes": 7558, "growth": [] } ], "imageFiles": [ { "path": "workspace/pet-shop-website/assets/logo.png", "bytes": 1571902 }, { "path": "artifacts/pet-shop-website/generated/image-2026-11-09T21-11-17-063Z-45739106.png", "bytes": 1571902 }, { "path": "artifacts/pet-shop-website/sessions/2026-11-09_202923_new-session/tool-0-img-0.png", "bytes": 1571902 } ], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 493580, "filePath": "pet-shop-website/workspace/index.html", "bytes": 5639, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 2080160, "filePath": "pet-shop-website/workspace/index.html", "bytes": 17247, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 2428139, "filePath": "pet-shop-website/workspace/index.html", "bytes": 7544, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 2678332, "filePath": "pet-shop-website/workspace/index.html", "bytes": 7554, "score": 3, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page" ], "failReason": null }, { "atMs": 2767033, "filePath": "pet-shop-website/workspace/index.html", "bytes": 7554, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null } ], "latest": { "filePath": "pet-shop-website/workspace/index.html", "bytes": 7554, "score": 5, "scoreMax": 5, "signals": [ "pet-vocab", "store-vocab", "structured-page", "working-image", "image-asset" ], "failReason": null, "runtimePassed": 1, "runtimeFailed": 0 } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T20:25:10.818Z] [trial] id=petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy scenario=petshop model=underdog-saluki-27b-iq2-mix", "[2026-11-09T20:25:10.818Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-petshop-xjYZqf", "[2026-11-09T20:25:10.818Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T20:25:10.818Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "[2026-11-09T20:25:10.818Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T20:25:10.821Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T20:25:10.822Z] [trial] sd-server=packages\\app\\native-bin\\win32-x64\\gezel-sd-server.exe", "[2026-11-09T20:25:10.827Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T20:25:10.829Z] [trial] sd-cpp source=\\.gezel-eval-cache\\engines\\sd-cpp\\models\\sdxl-lightning-4step (read-only)", "[2026-11-09T20:25:10.835Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-petshop-xjYZqf", "[2026-11-09T20:25:10.839Z] [trial] cloned sd-cpp/sdxl-lightning-4step from \\.gezel-eval-cache\\engines\\sd-cpp\\models\\sdxl-lightning-4step into \\AppData\\Local\\Temp\\gezel-eval-petshop-xjYZqf", "[2026-11-09T20:25:10.839Z] [trial] throughput-scaled ceiling: 120m → 182m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T20:25:16.297Z] [trial] daemon spawned pid=6796 port=https://127.0.0.1:51218", "[2026-11-09T20:25:17.262Z] [trial] provider=llama-cpp imageProvider=sd-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T20:25:17.279Z] [trial] meester=vanya", "[2026-11-09T20:25:17.432Z] [trial] sent prompt to meester (795 chars)", "[2026-11-09T20:25:17.490Z] [poll] initial digests hard=15579224a74f461d soft=4b7bc94d3dbd9ab7", "[2026-11-09T20:26:10.442Z] [poll] hard-progress: 15579224a74f461d → 258b34af499e8575 after 53s (workspaceProjects=2 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=88,stream=330) sniff=none)", "[2026-11-09T20:26:52.091Z] [poll] hard-progress: 258b34af499e8575 → d37dad6f4ff0e85c after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=184,stream=795) sniff=none)", "[2026-11-09T20:27:09.702Z] [poll] hard-progress: d37dad6f4ff0e85c → 07452fc100e061f7 after 18s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=211,stream=874) sniff=none)", "[2026-11-09T20:28:09.409Z] [poll] hard-progress: 07452fc100e061f7 → 6477ed819e731c06 after 60s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=255,stream=963) sniff=none)", "[2026-11-09T20:29:08.811Z] [poll] hard-progress: 6477ed819e731c06 → 93e99c4c1afce81e after 59s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=4,slot=394,stream=1710) sniff=none)", "[2026-11-09T20:29:26.773Z] [poll] hard-progress: 93e99c4c1afce81e → 3c6ad3569d6d18f6 after 18s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=3,tools=6,slot=426,stream=1871) sniff=none)", "[2026-11-09T20:33:18.608Z] [poll] hard-progress: 3c6ad3569d6d18f6 → 8e6b7fada3ddf5ac after 232s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=3,tools=7,slot=1013,stream=5047) sniff=none)", "[2026-11-09T20:33:24.607Z] [poll] hard-progress: 8e6b7fada3ddf5ac → 9bb8eacc0f5179e7 after 6s (workspaceProjects=3 sessions=5 daemon[src=service-telemetry](turns=3,tools=7,slot=1013,stream=5047) sniff=3/5639B)" ] }, "metrics": { "process": { "sampleCount": 463, "peakRssMb": 13399, "peakCpuPercent": 1245, "firstAtMs": 0, "lastAtMs": 2760551 }, "gpu": { "available": true, "sampleCount": 463, "peakUtilPercent": 100, "peakMemUsedMb": 10652, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 463, "peakUsedMb": 31506, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 121400, "totalOutputTokens": 18893, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 89665, "todayTokensCached": 66461, "todayTokensOut": 6038, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 89665, "totalTokensCached": 66461, "totalTokensOut": 6038, "totalCost": 0, "medianOutputTokensPerSec": 8.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.2, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9780, "outputTokens": 162, "cost": 0, "durationMs": 73399, "outputTokensPerSec": 8.223899284714866, "at": "2026-11-09T21:09:45.761Z" }, "lastUpdated": "2026-11-09T21:09:45.761Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:11:19.240Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2762961, "meanTokensPerSec": 8.17, "genTokensPerSec": 8.17, "genTokens": 18893, "promptTokensPerSec": 392.99, "peakRssMb": 13399, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 89665, "output": 6038 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T20:25:10.814Z", "finishedAt": "2026-11-09T21:11:32.304Z", "timeToFirstArtifactMs": 487687, "timeToLastArtifactWriteMs": 2674918, "timeToFirstTokenMs": 28689, "firstTurnTtftMs": 19110, "timeToFirstToolCallMs": 59307 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "pet-shop-website/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-09T20:26:47.187Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "9a43aeb6-301a-430a-b6a5-350bba964a92" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T20:25:16.432Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T20:25:16.333Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 5, "taskScoped": 1, "byGezel": { "darika": 1, "kumar": 1, "naoki": 1, "vanya": 2 }, "perTask": { "pet-shop-website/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 7, "maxContextFill": 0.18939208984375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.18939208984375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 6 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-petshop", "baselineCellId": "r1-saluki-cuda-core-petshop", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "scenarioId": "petshop", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "scored", "reviewedAt": "2026-11-09T21:51:04.649Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/workspace/pet-shop-website/index.html", "artifactBytes": 7558, "artifactSha256": "0d7672bfd2a7dd492f3cfba0e100cab268704f42627f8749682a2b4d266605ff", "axisSource": "evals/src/llm-judge.ts DEFAULT_HTML_AXES", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 6, "functionalCompleteness": 6, "codeQuality": 6, "polish": 4 }, "meanScore": 5.5, "justification": "Coherent responsive storefront; all navigation targets and the generated PNG load at desktop and 320px. The animal/paw logo is visibly distorted, with broad empty padding. Product and service copy is generic, contact details are placeholders, and the static page offers little polish beyond the brief.", "reviewBasis": "Read complete primary artifacts within the 20KiB cap, all declared companion outputs and relevant scenario contract. Inspected tool receipts and terminal facts. Browser checks/images or production-export probes are saved separately; no artifact or fixed evidence was changed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/workspace/pet-shop-website/index.html", "bytes": 7558, "sha256": "0d7672bfd2a7dd492f3cfba0e100cab268704f42627f8749682a2b4d266605ff", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 7558, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/workspace/pet-shop-website/assets/logo.png", "bytes": 1571902, "sha256": "0e2d9eac42d46ff90dd030d784eedaa3b3ef56b2703b0a95c4092d913cddcdae", "role": "declaredOutput", "status": "reviewed", "viewedImage": true } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T20:25:18.242Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T20:25:21.663Z INFO [chat] [llama-server] launch {\"pid\":42660,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}", "2026-11-09T21:11:21.616Z INFO [chat] [llama-server] launch {\"pid\":23492,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T21:10:48.739Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-42660-1794258648738\",\"pid\":42660,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"memory-pressure\",\"uptimeMs\":2727075,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 21272, "peakGpuMiB": 10652, "peakProcessRssMiB": 13399, "peakSystemRamMiB": 31506, "decodeTokensPerSec": 8.17, "prefillTokensPerSec": 392.99, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.18939208984375 }, "note": "Hybrid task placement, 39/64 dense FFN layers on GPU and 25 CPU. Whole-system memory includes other applications; shared GPU memory and paging are unmeasured. Multiple launches alone do not imply a native crash." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:11.682Z", "reviewId": "088", "cellId": "r3-saluki-cuda-core-petshop", "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "scenarioId": "petshop", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "baselineCellId": "r1-saluki-cuda-core-petshop", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "7797a3a94468547bf9db38b78b2412d005dd8aeb690066fc06df5ba57fe6d551", "evidenceSha256": "30a90d003a35eb6d6d4aaf93fb1dd8bccaea9d3dcc24afb8aebbc18ad7630723", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/workspace/pet-shop-website/index.html", "bytes": 7558, "sha256": "0d7672bfd2a7dd492f3cfba0e100cab268704f42627f8749682a2b4d266605ff", "shownBytes": 7558, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 5, "codeQuality": 6, "polish": 3 }, "justification": "Clean, coherent single page: sticky header, gradient hero, product cards, services, contact, responsive CSS variables. It's generic and text-only with no product imagery. The logo img is wired to assets/logo.png, but the alt text differs from the requested one. The PNG isn't shown, so generation is unverified.", "limitations": "Source-only review; assets/logo.png isn't in the evidence, so I can't confirm it exists or what it looks like. No rendering. Image-tool usage and the workspace process can't be checked.", "meanScore": 4.75, "durationMs": 15875, "costUsd": 0.025997600000000003, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5282, "cache_read_input_tokens": 856, "output_tokens": 478, "output_tokens_details": { "thinking_tokens": 156 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5282, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 478, "cache_read_input_tokens": 856, "cache_creation_input_tokens": 5282, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5282 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 478, "cacheReadInputTokens": 856, "cacheCreationInputTokens": 5282, "webSearchRequests": 0, "costUSD": 0.025997600000000003, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 156, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5282, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - petshop / underdog-saluki-27b-iq2-mix\n\n**Composite: 7.0 / 10** (band: needs-tuning)\n**Outcome: success** (duration: 46m 21s; budget used: 25.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 487687`; `facts.sniff.latest: {\"filePath\":\"pet-shop-website/workspace/index.html\",\"bytes\":7554,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fa...`; `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-website/assets/logo.png\",\"bytes\":1571902},{\"path\":\"artifacts/pet-shop-website/generated/image-2026-11-09T21-11-17-063Z-45739106.png\",\"bytes\":1571902...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2553562344739592`; `facts.toolUse.totalToolCalls: 32`; `facts.timing.timeToFirstArtifactMs: 487687`; `facts.outcome.durationMs: 2781490` |\n| Behavior soundness (15%) | 0 / 10 | At least one red-flag behavior fired or an expected role was missing. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: [\"image-generator\"]`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 0 = **7.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13399 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10652 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31506 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.17 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 121400 + 18893 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 19% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 6 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 487687`\n- `facts.sniff.latest: {\"filePath\":\"pet-shop-website/workspace/index.html\",\"bytes\":7554,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fa...`\n- `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-website/assets/logo.png\",\"bytes\":1571902},{\"path\":\"artifacts/pet-shop-website/generated/image-2026-11-09T21-11-17-063Z-45739106.png\",\"bytes\":1571902...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.2553562344739592`\n- `facts.toolUse.totalToolCalls: 32`\n- `facts.timing.timeToFirstArtifactMs: 487687`\n- `facts.outcome.durationMs: 2781490`\n\n### Behavior soundness - B_RED_FLAG_OR_MISSING_ROLE\n\nAt least one red-flag behavior fired or an expected role was missing.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: [\"image-generator\"]`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 487687`; `facts.sniff.latest: {\"filePath\":\"pet-shop-website/workspace/index.html\",\"bytes\":7554,\"score\":5,\"scoreMax\":5,\"signals\":[\"pet-vocab\",\"store-vocab\",\"structured-page\",\"working-image\",\"image-asset\"],\"fa...`; `facts.artifacts.imageFiles: [{\"path\":\"workspace/pet-shop-website/assets/logo.png\",\"bytes\":1571902},{\"path\":\"artifacts/pet-shop-website/generated/image-2026-11-09T21-11-17-063Z-45739106.png\",\"bytes\":1571902...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.2553562344739592`; `facts.toolUse.totalToolCalls: 32`; `facts.timing.timeToFirstArtifactMs: 487687`; `facts.outcome.durationMs: 2781490`\n- Behavior: At least one red-flag behavior fired or an expected role was missing. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: [\"image-generator\"]`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed composite **7/10**, pass; eligibility unchanged. The advisory mean is **5.5/10**. Coherent responsive storefront; all navigation targets and the generated PNG load at desktop and 320px. The animal/paw logo is visibly distorted, with broad empty padding. Product and service copy is generic, contact details are placeholders, and the static page offers little polish beyond the brief.\n\n| Advisory axis | Score |\n|---|---|\n| visualQuality | 6/10 |\n| functionalCompleteness | 6/10 |\n| codeQuality | 6/10 |\n| polish | 4/10 |\n\n## Positives\n\nThe final single-page site references a real generated PNG; the image loads at 1536x1536, all anchors resolve and the 320px layout has no horizontal overflow (sniff.latest.score=5/5, runtimePassed=1; advisory browser evidence).\n\n## Negatives\n\nTen write_file attempts to a raster path were rejected before generate_image succeeded; six repeat aborts and 32 calls accompany 46.36 minutes of work (tool receipts; continuity.budget.toolRepeatAborts=6; toolUse.totalToolCalls=32). Fixed behavior is zero because team.missingExpectedRoles contains image-generator, despite an Illustrator and a successful image tool call; preserve this naming-sensitive metric and audit its role mapping separately. Three model validate calls did not run a browser, but the terminal harness did.\n\n## Performance and fit\n\nDecode 8.17 tok/s; prefill 392.99 tok/s. Peak GPU 10652/12227 MiB, process-tree RSS 13399 MiB, system RAM 31506/31865 MiB (facts.perf). Context 65536, 1 slot, f16; max observed released slot 21272 tokens. Explicit native/OOM/context fault lines: 0. Hybrid placement is not full GPU residency, and shared GPU memory/paging remain unmeasured. Petshop includes image-engine use; a second chat launch alone is not classified as a crash.\n\n## Tactical experiment — not applied\n\nExperiment only: make ../gilde/data/gezel-templates/il/illustrator/versions/1.0.0/about.md require the available image-generation tool as the first action for requested raster generation, then validate on unrelated icons and report illustrations. Evidence is ten rejected raster writes and one eventual successful generate_image, not model size.\n\n## Strategic experiment — not applied\n\nExperiment only: persist repeated binary-write failure identity across delegated sessions in packages/service/src/providers/unresolved-tool-failure-ledger.ts, using packages/service/src/fs/binary-text.ts diagnostics to route one corrective handoff. Separately audit role aliases in evals/src/bin/score-trial.ts; never silently rescore this campaign.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/facts.json): pass, fixed 7, 62.91 min, eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/facts.json): pass, fixed 7.5, 81.05 min, eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/facts.json): pass, fixed 8.5, 2.46 min, eligible.\n\nUnrelated controls include original R2 codemod sweep (pass 8.5) and constrained communications (fail 6). A successful artifact or compile gate does not establish broad workflow or output quality. The proposals target reusable validation, state boundaries and tool recovery, with no fixture-specific global hint. Original/continuation GEZEL_MXC_EXEC is an explicit comparison boundary; other-model/native history and differing automatic timeout scaling prevent causal speed/quality attribution.\n\n## Recommended next experiment\n\nEvaluate the proposed diagnostic on one unrelated task type and record advisory defects, repair turns and false positives without modifying this campaign's fixed evidence.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, historical-comparison.json, plus advisory-ui-checks-20261109 or advisory production-export checks where present.\n" }, { "cellId": "r3-saluki-cuda-core-tankcombat", "baselineCellId": "r1-saluki-cuda-core-tankcombat", "suite": "core", "scenario": "tankcombat", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T21:14:42.226Z", "finishedAt": "2026-11-09T21:22:47.099Z", "durationMs": 484873, "success": true, "reason": "tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": { "gameHtmlMinBytes": 4096 }, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 470861 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 8751, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": 3, "runtimeFailed": 0 } }, { "source": "facts", "path": "sniff.latest.bytes", "value": 8751 } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.04451403509561135 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 470861 }, { "source": "facts", "path": "outcome.durationMs", "value": 484873 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "scenarioId": "tankcombat", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 79, "peakRssMb": 10711, "peakCpuPercent": 1248.7, "firstAtMs": 0, "lastAtMs": 465887 }, "gpu": { "available": true, "sampleCount": 79, "peakUtilPercent": 95, "peakMemUsedMb": 10634, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 79, "peakUsedMb": 29090, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12703, "totalOutputTokens": 3634, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7599, "todayTokensCached": 7114, "todayTokensOut": 145, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7599, "totalTokensCached": 7114, "totalTokensOut": 145, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7599, "cachedInputTokens": 7114, "outputTokens": 145, "cost": 0, "durationMs": 95309, "outputTokensPerSec": 8.462332980579417, "at": "2026-11-09T21:16:27.112Z" }, "lastUpdated": "2026-11-09T21:16:27.112Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:22:37.792Z" } }, "source": "engine-log" }, "derived": { "durationMs": 469906, "meanTokensPerSec": 8.54, "genTokensPerSec": 8.54, "genTokens": 3634, "promptTokensPerSec": 578.85, "peakRssMb": 10711, "peakGpuUtilPercent": 95, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7599, "output": 145 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tank-combat-arcade/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-09T21:16:08.217Z", "completedAt": "2026-11-09T21:22:33.939Z", "ms": 385722, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "31206847-5310-426d-a096-d4db2c17e2d2" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:14:47.862Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:14:47.744Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 385722 }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "themistocles": 2, "zephyr": 1 }, "perTask": { "tank-combat-arcade/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.09539794921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.09539794921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "durationMs": 484873, "timeoutMs": 10892587, "budgetUsedFraction": 0.04451403509561135 }, "timing": { "startedAt": "2026-11-09T21:14:42.226Z", "finishedAt": "2026-11-09T21:22:47.099Z", "timeToFirstArtifactMs": 470861, "timeToLastArtifactWriteMs": 470903, "timeToFirstTokenMs": 28541, "firstTurnTtftMs": 18964, "timeToFirstToolCallMs": 86052 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "start_work": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/tank-combat-arcade/index.html", "finalBytes": 8751, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 472574, "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 8751, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null } ], "latest": { "filePath": "tank-combat-arcade/workspace/index.html", "bytes": 8751, "score": 9, "scoreMax": null, "signals": [ "tank-vocab", "render-surface", "keyboard-input", "combat", "game-loop", "gameplay", "js-parses", "js-size-ok", "html-size-ok" ], "failReason": null, "runtimePassed": 3, "runtimeFailed": 0 } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 472984, "kind": "structured", "gezel": "zephyr", "question": "Your web page is finished. - **`index.html`**", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-09T21:14:42.231Z] [trial] id=tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8 scenario=tankcombat model=underdog-saluki-27b-iq2-mix", "[2026-11-09T21:14:42.231Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-tankcombat-GsWZRx", "[2026-11-09T21:14:42.231Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "[2026-11-09T21:14:42.231Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T21:14:42.231Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T21:14:42.235Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T21:14:42.239Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T21:14:42.246Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-tankcombat-GsWZRx", "[2026-11-09T21:14:42.247Z] [trial] throughput-scaled ceiling: 120m → 182m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T21:14:47.830Z] [trial] daemon spawned pid=50312 port=https://127.0.0.1:59973", "[2026-11-09T21:14:48.763Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T21:14:48.779Z] [trial] meester=zephyr", "[2026-11-09T21:14:48.930Z] [trial] sent prompt to meester (1688 chars)", "[2026-11-09T21:14:48.988Z] [poll] initial digests hard=c0cb1b949dded876 soft=4c8569e73e4992b8", "[2026-11-09T21:16:11.969Z] [poll] hard-progress: c0cb1b949dded876 → 03a7c3926c8fa4ea after 83s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=167,stream=751) sniff=none)", "[2026-11-09T21:16:29.808Z] [poll] hard-progress: 03a7c3926c8fa4ea → 26c14989c9f6e0bd after 18s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=1,slot=210,stream=874) sniff=none)", "[2026-11-09T21:20:34.012Z] [poll] hard-progress: 26c14989c9f6e0bd → 29cc0c2669a7b210 after 244s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=1,slot=864,stream=4501) sniff=none)", "[2026-11-09T21:22:37.351Z] [poll] terminal: success=true reason=tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))", "[2026-11-09T21:22:37.847Z] [capture] dumped 3 session(s)", "[2026-11-09T21:22:37.852Z] [capture] snapshotted telemetry for 3 session(s)", "[2026-11-09T21:22:37.929Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-09T21:22:47.098Z] [trial] daemon shut down", "[2026-11-09T21:22:47.108Z] [trial] PASS duration=484873ms reason=tank-combat-arcade/workspace/index.html passed sniff (signals: tank-vocab, render-surface, keyboard-input, combat, game-loop, gameplay, js-parses, js-size-ok, html-size-ok) + runtime (3 assertion(s))" ] }, "metrics": { "process": { "sampleCount": 79, "peakRssMb": 10711, "peakCpuPercent": 1248.7, "firstAtMs": 0, "lastAtMs": 465887 }, "gpu": { "available": true, "sampleCount": 79, "peakUtilPercent": 95, "peakMemUsedMb": 10634, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 79, "peakUsedMb": 29090, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 12703, "totalOutputTokens": 3634, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 7599, "todayTokensCached": 7114, "todayTokensOut": 145, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 7599, "totalTokensCached": 7114, "totalTokensOut": 145, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7599, "cachedInputTokens": 7114, "outputTokens": 145, "cost": 0, "durationMs": 95309, "outputTokensPerSec": 8.462332980579417, "at": "2026-11-09T21:16:27.112Z" }, "lastUpdated": "2026-11-09T21:16:27.112Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:22:37.792Z" } }, "source": "engine-log" }, "derived": { "durationMs": 469906, "meanTokensPerSec": 8.54, "genTokensPerSec": 8.54, "genTokens": 3634, "promptTokensPerSec": 578.85, "peakRssMb": 10711, "peakGpuUtilPercent": 95, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 7599, "output": 145 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T21:14:42.226Z", "finishedAt": "2026-11-09T21:22:47.099Z", "timeToFirstArtifactMs": 470861, "timeToLastArtifactWriteMs": 470903, "timeToFirstTokenMs": 28541, "firstTurnTtftMs": 18964, "timeToFirstToolCallMs": 86052 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "tank-combat-arcade/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-09T21:16:08.217Z", "completedAt": "2026-11-09T21:22:33.939Z", "ms": 385722, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "31206847-5310-426d-a096-d4db2c17e2d2" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:14:47.862Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:14:47.744Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 385722 }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "themistocles": 2, "zephyr": 1 }, "perTask": { "tank-combat-arcade/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.09539794921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.09539794921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-tankcombat", "baselineCellId": "r1-saluki-cuda-core-tankcombat", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "scenarioId": "tankcombat", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "scored", "reviewedAt": "2026-11-09T21:51:04.704Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/workspace/tank-combat-arcade/index.html", "artifactBytes": 8751, "artifactSha256": "1a65a406515adcda74fc10b0075669a872900b6d94dfc9d510008d4786cdb0cf", "axisSource": "evals/src/llm-judge.ts DEFAULT_HTML_AXES", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "scoreAxes": { "visualQuality": 5, "functionalCompleteness": 4, "codeQuality": 4, "polish": 3 }, "meanScore": 4, "justification": "Movement, firing, enemy spawning, scoring, damage and restart work. A controlled final-life hit with another bullet present makes update throw after reset clears the array, which would stop the animation loop. Motion is frame-rate dependent; narrow views distort the arena. Bare visuals and abrupt resets limit polish.", "reviewBasis": "Read complete primary artifacts within the 20KiB cap, all declared companion outputs and relevant scenario contract. Inspected tool receipts and terminal facts. Browser checks/images or production-export probes are saved separately; no artifact or fixed evidence was changed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/workspace/tank-combat-arcade/index.html", "bytes": 8751, "sha256": "1a65a406515adcda74fc10b0075669a872900b6d94dfc9d510008d4786cdb0cf", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 8751, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T21:14:49.671Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T21:14:53.045Z INFO [chat] [llama-server] launch {\"pid\":36352,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T21:22:46.346Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-36352-1794259366346\",\"pid\":36352,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":473301,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 8109, "peakGpuMiB": 10634, "peakProcessRssMiB": 10711, "peakSystemRamMiB": 29090, "decodeTokensPerSec": 8.54, "prefillTokensPerSec": 578.85, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.09539794921875 }, "note": "Hybrid task placement, 39/64 dense FFN layers on GPU and 25 CPU. Whole-system memory includes other applications; shared GPU memory and paging are unmeasured. Multiple launches alone do not imply a native crash." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:16.704Z", "reviewId": "089", "cellId": "r3-saluki-cuda-core-tankcombat", "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "scenarioId": "tankcombat", "status": "scored", "axes": [ { "name": "visualQuality", "description": "Does it look like a real product, or a debug page?" }, { "name": "functionalCompleteness", "description": "Does it implement the user's stated brief?" }, { "name": "codeQuality", "description": "Would a developer reading the source be embarrassed?" }, { "name": "polish", "description": "Are there surprising / delightful touches?" } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "baselineCellId": "r1-saluki-cuda-core-tankcombat", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "ff8b50aab66b83dcf90ce7bc29ebccc959ca8da99e76753dad0fc6e3a98f35b7", "evidenceSha256": "546e7458a1017e2d08b477b24878ef4253e7702ba500522a7f536bea4355e9d5", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/workspace/tank-combat-arcade/index.html", "bytes": 8751, "sha256": "1a65a406515adcda74fc10b0075669a872900b6d94dfc9d510008d4786cdb0cf", "shownBytes": 8751, "truncated": false } ], "contextFiles": [], "scoreAxes": { "visualQuality": 4, "functionalCompleteness": 6, "codeQuality": 5, "polish": 3 }, "justification": "Single-file shape is correct. It has WASD/arrow movement, Space fire, spawning enemy tanks that shoot, collisions, an on-screen score, lives, restart, and input exposed via gameState and dataset.inputTick. Visuals are plain: a grid with square tanks and a small HUD. There is no game-over screen, and the code has dead variables and an angle-lerp wrap bug.", "limitations": "Source-only review with no rendering or runtime execution. The angle-wrap bug, canvas CSS stretching, and gameplay feel are inferred from the code and were not observed.", "meanScore": 4.5, "durationMs": 19438, "costUsd": 0.0332936, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6636, "cache_read_input_tokens": 856, "output_tokens": 666, "output_tokens_details": { "thinking_tokens": 317 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6636, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 666, "cache_read_input_tokens": 856, "cache_creation_input_tokens": 6636, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6636 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 666, "cacheReadInputTokens": 856, "cacheCreationInputTokens": 6636, "webSearchRequests": 0, "costUSD": 0.0332936, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 317, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6636, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - tankcombat / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 8m 5s; budget used: 4.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 470861`; `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":8751,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"game...`; `facts.sniff.latest.bytes: 8751` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.04451403509561135`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 470861`; `facts.outcome.durationMs: 484873` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10711 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 95% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10634 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29090 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.54 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 12703 + 3634 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 386s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 10% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 470861`\n- `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":8751,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"game...`\n- `facts.sniff.latest.bytes: 8751`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.04451403509561135`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 470861`\n- `facts.outcome.durationMs: 484873`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 470861`; `facts.sniff.latest: {\"filePath\":\"tank-combat-arcade/workspace/index.html\",\"bytes\":8751,\"score\":9,\"scoreMax\":null,\"signals\":[\"tank-vocab\",\"render-surface\",\"keyboard-input\",\"combat\",\"game-loop\",\"game...`; `facts.sniff.latest.bytes: 8751`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.04451403509561135`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 470861`; `facts.outcome.durationMs: 484873`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed composite **10/10**, pass; eligibility unchanged. The advisory mean is **4/10**. Movement, firing, enemy spawning, scoring, damage and restart work. A controlled final-life hit with another bullet present makes update throw after reset clears the array, which would stop the animation loop. Motion is frame-rate dependent; narrow views distort the arena. Bare visuals and abrupt resets limit polish.\n\n| Advisory axis | Score |\n|---|---|\n| visualQuality | 5/10 |\n| functionalCompleteness | 4/10 |\n| codeQuality | 4/10 |\n| polish | 3/10 |\n\n## Positives\n\nThe game ships in two tool calls and passes three standard runtime assertions (toolUse.totalToolCalls=2; sniff.latest.runtimePassed=3). Independent browser checks confirm actual movement, firing and enemy presence; controlled states confirm scoring and damage.\n\n## Negatives\n\nThe controlled final-life/multiple-bullet case throws Cannot read properties of undefined (reading x): reset empties bullets while update retains the old reverse-loop index. This exception would escape the ordinary loop before scheduling its next frame. The fixed runtime gate does not exercise this transition. Movement/bullets use per-frame increments while cooldowns use dt, and CSS squashes the fixed-height arena at narrow widths (advisory-ui-checks-20261109/checks.json; reviewed source).\n\n## Performance and fit\n\nDecode 8.54 tok/s; prefill 578.85 tok/s. Peak GPU 10634/12227 MiB, process-tree RSS 10711 MiB, system RAM 29090/31865 MiB (facts.perf). Context 65536, 1 slot, f16; max observed released slot 8109 tokens. Explicit native/OOM/context fault lines: 0. Hybrid placement is not full GPU residency, and shared GPU memory/paging remain unmeasured. Petshop includes image-engine use; a second chat launch alone is not classified as a crash.\n\n## Tactical experiment — not applied\n\nExperiment only: add one terminal-state and restart check to the reusable browser-game validation guidance in ../gilde/data/gezel-templates/bu/builder/versions/1.5.1/about.md. Measure end-state exceptions across unrelated games and interactive simulations; avoid expected DOM strings or fixture recipes.\n\n## Strategic experiment — not applied\n\nExperiment only: extend packages/mcp/src/validate.ts with a separate optional interaction trace that records post-terminal page errors and stale animation loops. Keep campaign scores/gates frozen; validate the diagnostic on unrelated stateful widgets.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/facts.json): pass, fixed 10, 7.54 min, eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/facts.json): pass, fixed 10, 30.01 min, eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/facts.json): pass, fixed 10, 0.86 min, eligible.\n\nUnrelated controls include original R2 codemod sweep (pass 8.5) and constrained communications (fail 6). A successful artifact or compile gate does not establish broad workflow or output quality. The proposals target reusable validation, state boundaries and tool recovery, with no fixture-specific global hint. Original/continuation GEZEL_MXC_EXEC is an explicit comparison boundary; other-model/native history and differing automatic timeout scaling prevent causal speed/quality attribution.\n\n## Recommended next experiment\n\nEvaluate the proposed diagnostic on one unrelated task type and record advisory defects, repair turns and false positives without modifying this campaign's fixed evidence.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, historical-comparison.json, plus advisory-ui-checks-20261109 or advisory production-export checks where present.\n" }, { "cellId": "r3-saluki-cuda-core-schema-migration", "baselineCellId": "r1-saluki-cuda-core-schema-migration", "suite": "core", "scenario": "schema-migration", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T21:25:46.084Z", "finishedAt": "2026-11-09T21:34:18.578Z", "durationMs": 512494, "success": true, "reason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 85132 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "schema-migration", "bytes": 6958, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.16131356419017281 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 14 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 85132 }, { "source": "facts", "path": "outcome.durationMs", "value": 512494 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "scenarioId": "schema-migration", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 85, "peakRssMb": 14150, "peakCpuPercent": 4515.8, "firstAtMs": 0, "lastAtMs": 500372 }, "gpu": { "available": true, "sampleCount": 85, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 85, "peakUsedMb": 31063, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 66119, "totalOutputTokens": 2674, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 57523, "todayTokensCached": 11574, "todayTokensOut": 2425, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 57523, "totalTokensCached": 11574, "totalTokensOut": 2425, "totalCost": 0, "medianOutputTokensPerSec": 9.1, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.1, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11520, "outputTokens": 757, "cost": 0, "durationMs": 152042, "outputTokensPerSec": 9.036282178995698, "at": "2026-11-09T21:34:08.699Z" }, "lastUpdated": "2026-11-09T21:34:08.699Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:34:13.911Z" } }, "source": "engine-log" }, "derived": { "durationMs": 502220, "meanTokensPerSec": 9.16, "genTokensPerSec": 9.16, "genTokens": 2674, "promptTokensPerSec": 406.26, "peakRssMb": 14150, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 57523, "output": 2425 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:25:51.690Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:25:51.581Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "riley": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 15, "maxContextFill": 0.169708251953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.169708251953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all 7 signals firing including tsc-clean (signals: types-updated, store-updated, handlers-updated, migrate-function, tests-present, migration-doc, tsc-clean)", "durationMs": 512494, "timeoutMs": 3177005, "budgetUsedFraction": 0.16131356419017281 }, "timing": { "startedAt": "2026-11-09T21:25:46.084Z", "finishedAt": "2026-11-09T21:34:18.578Z", "timeToFirstArtifactMs": 85132, "timeToLastArtifactWriteMs": 502615, "timeToFirstTokenMs": 24483, "firstTurnTtftMs": 14764, "timeToFirstToolCallMs": 35844 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 14, "byTool": { "read_files": 1, "write_file": 4, "replace_in_file": 4, "read_file": 4, "stat": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 11 }, "sniff": { "progression": [ { "atMs": 7119, "filePath": "schema-migration", "bytes": 1659, "score": 0, "scoreMax": 7, "signals": [], "failReason": "types.ts still declares name or is missing required string firstName/lastName fields on User and CreateUserInput" }, { "atMs": 90092, "filePath": "schema-migration", "bytes": 2897, "score": 0, "scoreMax": 7, "signals": [], "failReason": "types.ts still declares name or is missing required string firstName/lastName fields on User and CreateUserInput" }, { "atMs": 120373, "filePath": "schema-migration", "bytes": 2934, "score": 2, "scoreMax": 7, "signals": [ "types-updated", "migrate-function" ], "failReason": "store.ts still uses a live name property or is missing firstName/lastName" }, { "atMs": 155224, "filePath": "schema-migration", "bytes": 2970, "score": 2, "scoreMax": 7, "signals": [ "types-updated", "migrate-function" ], "failReason": "store.ts still uses a live name property or is missing firstName/lastName" }, { "atMs": 191338, "filePath": "schema-migration", "bytes": 3031, "score": 3, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "migrate-function" ], "failReason": "handlers.ts unmet subcontracts: format-display-full-name (return firstName + space + lastName from formatDisplayName); render-card-full-name (include firstName + space + lastName in the rendered card); summary-full-name (include each mapped user firstName + space + lastName); no-legacy-name-access (remove every live .name property access)" }, { "atMs": 226849, "filePath": "schema-migration", "bytes": 3099, "score": 4, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function" ], "failReason": "tests/migrate.test.ts not present yet" }, { "atMs": 269064, "filePath": "schema-migration", "bytes": 3047, "score": 4, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function" ], "failReason": "tests/migrate.test.ts not present yet" }, { "atMs": 352523, "filePath": "schema-migration", "bytes": 4331, "score": 5, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present" ], "failReason": "MIGRATION.md not present yet" }, { "atMs": 507351, "filePath": "schema-migration", "bytes": 6958, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } ], "latest": { "filePath": "schema-migration", "bytes": 6958, "score": 7, "scoreMax": 7, "signals": [ "types-updated", "store-updated", "handlers-updated", "migrate-function", "tests-present", "migration-doc", "tsc-clean" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T21:25:46.089Z] [trial] id=schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203 scenario=schema-migration model=underdog-saluki-27b-iq2-mix", "[2026-11-09T21:25:46.089Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "[2026-11-09T21:25:46.089Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-schema-migration-8i2N5N", "[2026-11-09T21:25:46.089Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T21:25:46.089Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T21:25:46.092Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T21:25:46.096Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T21:25:46.101Z] [trial] throughput-scaled ceiling: 35m → 53m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T21:25:46.101Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-schema-migration-8i2N5N", "[2026-11-09T21:25:51.646Z] [trial] daemon spawned pid=35604 port=https://127.0.0.1:58411", "[2026-11-09T21:25:52.582Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T21:25:52.600Z] [trial] running scenario setup hook", "[2026-11-09T21:25:52.600Z] [trial] meester=nadia", "[2026-11-09T21:25:53.088Z] [trial] scenario setup complete", "[2026-11-09T21:25:53.090Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T21:25:53.168Z] [poll] initial digests hard=fec515136db32303 soft=52ecda973ca9eba3", "[2026-11-09T21:25:53.374Z] [poll] hard-progress: fec515136db32303 → bbc34d5a8ca25dab after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/1659B)", "[2026-11-09T21:26:22.482Z] [poll] hard-progress: bbc34d5a8ca25dab → 1e008c74eb29bb37 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=37,stream=137) sniff=0/1659B)", "[2026-11-09T21:27:16.231Z] [poll] hard-progress: 1e008c74eb29bb37 → d077b451408e1539 after 54s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=2,slot=164,stream=795) sniff=0/2897B)", "[2026-11-09T21:27:46.571Z] [poll] hard-progress: d077b451408e1539 → e82be4d9f78178e3 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=3,slot=213,stream=1015) sniff=2/2934B)", "[2026-11-09T21:28:21.420Z] [poll] hard-progress: e82be4d9f78178e3 → 896b32f75000e118 after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=4,slot=269,stream=1253) sniff=2/2970B)", "[2026-11-09T21:28:33.512Z] [poll] hard-progress: 896b32f75000e118 → f7451d6744a5160f after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=5,slot=278,stream=1269) sniff=2/2970B)", "[2026-11-09T21:28:57.529Z] [poll] hard-progress: f7451d6744a5160f → 7d868ea340d74dae after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=5,tools=6,slot=312,stream=1403) sniff=3/3031B)", "[2026-11-09T21:29:33.117Z] [poll] hard-progress: 7d868ea340d74dae → e8579128624e276c after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=6,tools=7,slot=395,stream=1823) sniff=4/3099B)", "[2026-11-09T21:29:44.982Z] [poll] hard-progress: e8579128624e276c → d3699ce22c5d67c7 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=6,tools=8,slot=403,stream=1841) sniff=4/3099B)" ] }, "metrics": { "process": { "sampleCount": 85, "peakRssMb": 14150, "peakCpuPercent": 4515.8, "firstAtMs": 0, "lastAtMs": 500372 }, "gpu": { "available": true, "sampleCount": 85, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 85, "peakUsedMb": 31063, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 66119, "totalOutputTokens": 2674, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 8, "todayTokensIn": 57523, "todayTokensCached": 11574, "todayTokensOut": 2425, "todayCost": 0, "totalTurns": 8, "totalTokensIn": 57523, "totalTokensCached": 11574, "totalTokensOut": 2425, "totalCost": 0, "medianOutputTokensPerSec": 9.1, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.1, "turns": 8 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11520, "outputTokens": 757, "cost": 0, "durationMs": 152042, "outputTokensPerSec": 9.036282178995698, "at": "2026-11-09T21:34:08.699Z" }, "lastUpdated": "2026-11-09T21:34:08.699Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:34:13.911Z" } }, "source": "engine-log" }, "derived": { "durationMs": 502220, "meanTokensPerSec": 9.16, "genTokensPerSec": 9.16, "genTokens": 2674, "promptTokensPerSec": 406.26, "peakRssMb": 14150, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 57523, "output": 2425 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T21:25:46.084Z", "finishedAt": "2026-11-09T21:34:18.578Z", "timeToFirstArtifactMs": 85132, "timeToLastArtifactWriteMs": 502615, "timeToFirstTokenMs": 24483, "firstTurnTtftMs": 14764, "timeToFirstToolCallMs": 35844 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:25:51.690Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:25:51.581Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "riley": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 15, "maxContextFill": 0.169708251953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.169708251953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-schema-migration", "baselineCellId": "r1-saluki-cuda-core-schema-migration", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "scenarioId": "schema-migration", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "reviewedAt": "2026-11-09T21:51:04.763Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/types.ts", "artifactBytes": 412, "artifactSha256": "82e1cb9c5cf8038253d7b9745933e9a7aff74ecfc918380eb303ca7b9880992a", "axisSource": "evals/src/scenarios/schema-migration.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. All six deliverables are present. The helper splits on the first space, removes name and preserves extra fields in direct probes; all three handlers use the new names. Seeded compiler/runtime gates passed. The old README and comments remain stale, and single-word display names retain a trailing space.", "reviewBasis": "Read complete primary artifacts within the 20KiB cap, all declared companion outputs and relevant scenario contract. Inspected tool receipts and terminal facts. Browser checks/images or production-export probes are saved separately; no artifact or fixed evidence was changed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/types.ts", "bytes": 412, "sha256": "82e1cb9c5cf8038253d7b9745933e9a7aff74ecfc918380eb303ca7b9880992a", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 412, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/store.ts", "bytes": 835, "sha256": "6c877c17a044a87df7ed7af1488eb1f879d841902fd183eba20260a8fe34c735", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 835, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/handlers.ts", "bytes": 614, "sha256": "f151082616220d177a6e1bc1d525727b820376df24996db8e901924a3d2b1b85", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 614, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/migrate.ts", "bytes": 1186, "sha256": "21d86a7450d0fee85195b4705ef4cd0ea1a751975a476e418d91ad73803c3793", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 1186, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/tests/migrate.test.ts", "bytes": 1284, "sha256": "61dfe278d9d8c0fb5000f7f1d0bc91cc47b84ea476f53c1ffe4faf0aec075b7c", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 1284, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/MIGRATION.md", "bytes": 2631, "sha256": "ef9269029a603bcd84a9add231ba2c97c8e5d3127ead63815d29c780038fd74a", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 2631, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T21:25:53.927Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T21:25:57.097Z INFO [chat] [llama-server] launch {\"pid\":21440,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T21:34:17.811Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-21440-1794260057811\",\"pid\":21440,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":500715,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 12276, "peakGpuMiB": 10644, "peakProcessRssMiB": 14150, "peakSystemRamMiB": 31063, "decodeTokensPerSec": 9.16, "prefillTokensPerSec": 406.26, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.169708251953125 }, "note": "Hybrid task placement, 39/64 dense FFN layers on GPU and 25 CPU. Whole-system memory includes other applications; shared GPU memory and paging are unmeasured. Multiple launches alone do not imply a native crash." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:13.931Z", "reviewId": "090", "cellId": "r3-saluki-cuda-core-schema-migration", "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "scenarioId": "schema-migration", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "baselineCellId": "r1-saluki-cuda-core-schema-migration", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "fc0cbf380a53e82ad7c023e591c5aa59a577a1220c19cf0a377479a881326696", "evidenceSha256": "1ad8aa0424f799f93bd25fceafad69d581fe17484739117d5449999ea619fa08", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/types.ts", "bytes": 412, "sha256": "82e1cb9c5cf8038253d7b9745933e9a7aff74ecfc918380eb303ca7b9880992a", "shownBytes": 412, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/store.ts", "bytes": 835, "sha256": "6c877c17a044a87df7ed7af1488eb1f879d841902fd183eba20260a8fe34c735", "shownBytes": 835, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/handlers.ts", "bytes": 614, "sha256": "f151082616220d177a6e1bc1d525727b820376df24996db8e901924a3d2b1b85", "shownBytes": 614, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/migrate.ts", "bytes": 1186, "sha256": "21d86a7450d0fee85195b4705ef4cd0ea1a751975a476e418d91ad73803c3793", "shownBytes": 1186, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/tests/migrate.test.ts", "bytes": 1284, "sha256": "61dfe278d9d8c0fb5000f7f1d0bc91cc47b84ea476f53c1ffe4faf0aec075b7c", "shownBytes": 1284, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/MIGRATION.md", "bytes": 2631, "sha256": "ef9269029a603bcd84a9add231ba2c97c8e5d3127ead63815d29c780038fd74a", "shownBytes": 2631, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes are declared. Qualitatively, all six target files are present and consistent. types, store and handlers use firstName/lastName. migrateUser splits on the first space and preserves id, email and other fields via rest spread. There are three tests (normal, one-word, multi-word). MIGRATION.md is 2.6 KB.", "limitations": "Source-only review. Typechecks and vitest were not run, so passing is unverified. Stale comments remain in store.ts and handlers.ts that still mention `name`. Tests use .ts import extensions, which depends on tsconfig, and tsconfig.json was not shown.", "meanScore": null, "durationMs": 14366, "costUsd": 0.024315999999999997, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5258, "cache_read_input_tokens": 700, "output_tokens": 321, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5258, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 321, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 5258, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5258 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 321, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 5258, "webSearchRequests": 0, "costUSD": 0.024315999999999997, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5258, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - schema-migration / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 8m 32s; budget used: 16.1%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 85132`; `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":6958,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.16131356419017281`; `facts.toolUse.totalToolCalls: 14`; `facts.timing.timeToFirstArtifactMs: 85132`; `facts.outcome.durationMs: 512494` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14150 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31063 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.16 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 66119 + 2674 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 17% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 85132`\n- `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":6958,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.16131356419017281`\n- `facts.toolUse.totalToolCalls: 14`\n- `facts.timing.timeToFirstArtifactMs: 85132`\n- `facts.outcome.durationMs: 512494`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 85132`; `facts.sniff.latest: {\"filePath\":\"schema-migration\",\"bytes\":6958,\"score\":7,\"scoreMax\":7,\"signals\":[\"types-updated\",\"store-updated\",\"handlers-updated\",\"migrate-function\",\"tests-present\",\"migration-do...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.16131356419017281`; `facts.toolUse.totalToolCalls: 14`; `facts.timing.timeToFirstArtifactMs: 85132`; `facts.outcome.durationMs: 512494`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed composite **9.5/10**, pass; eligibility unchanged. The advisory status is **notApplicable** (no declared axes). No declared axes. All six deliverables are present. The helper splits on the first space, removes name and preserves extra fields in direct probes; all three handlers use the new names. Seeded compiler/runtime gates passed. The old README and comments remain stale, and single-word display names retain a trailing space.\n\n\n\n## Positives\n\nAll seven fixed signals fire, including strict typechecking and migration behavior (sniff.latest.score=7/7). Direct production-export probes confirm two-word, one-word, multi-word, repeated-space and empty strings, preservation of extra properties, removal of name and no input mutation (advisory-migration-checks.json).\n\n## Negatives\n\nThe unchanged README and store/handler comments still describe the old shape. Handler string interpolation leaves a trailing space when lastName is empty. Authored tests cover the three requested name shapes; model-authored tests were not executed by the reviewer, and the runner uses its documented typed/harness-owned checks. These limits do not change the fixed 9.5.\n\n## Performance and fit\n\nDecode 9.16 tok/s; prefill 406.26 tok/s. Peak GPU 10644/12227 MiB, process-tree RSS 14150 MiB, system RAM 31063/31865 MiB (facts.perf). Context 65536, 1 slot, f16; max observed released slot 12276 tokens. Explicit native/OOM/context fault lines: 0. Hybrid placement is not full GPU residency, and shared GPU memory/paging remain unmeasured. Petshop includes image-engine use; a second chat launch alone is not classified as a crash.\n\n## Tactical experiment — not applied\n\nExperiment only: add one documentation/call-site consistency pass to the general coding checklist in ../gilde/data/gezel-templates/bu/builder/versions/1.5.1/about.md. Measure stale public-shape descriptions across unrelated schema changes and codemods; do not teach the specific field rename.\n\n## Strategic experiment — not applied\n\nExperiment only: add optional changed-symbol documentation and output-format findings to packages/mcp/src/validate.ts. Keep semantic migration probes separate from textual documentation checks and measure false positives on unrelated refactors.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/facts.json): pass, fixed 9.5, 8.81 min, eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/facts.json): pass, fixed 9.5, 27.47 min, eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/facts.json): pass, fixed 9, 3.49 min, eligible.\n\nUnrelated controls include original R2 codemod sweep (pass 8.5) and constrained communications (fail 6). A successful artifact or compile gate does not establish broad workflow or output quality. The proposals target reusable validation, state boundaries and tool recovery, with no fixture-specific global hint. Original/continuation GEZEL_MXC_EXEC is an explicit comparison boundary; other-model/native history and differing automatic timeout scaling prevent causal speed/quality attribution.\n\n## Recommended next experiment\n\nEvaluate the proposed diagnostic on one unrelated task type and record advisory defects, repair turns and false positives without modifying this campaign's fixed evidence.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, historical-comparison.json, plus advisory-ui-checks-20261109 or advisory production-export checks where present.\n" }, { "cellId": "r3-saluki-cuda-core-failing-tests-spec", "baselineCellId": "r1-saluki-cuda-core-failing-tests-spec", "suite": "core", "scenario": "failing-tests-spec", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T21:37:05.191Z", "finishedAt": "2026-11-09T21:38:51.740Z", "durationMs": 106549, "success": true, "reason": "vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 99882 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "failing-tests-spec", "bytes": 1766, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.046952591758916556 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 3 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 99882 }, { "source": "facts", "path": "outcome.durationMs", "value": 106549 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "scenarioId": "failing-tests-spec", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 17, "peakRssMb": 9932, "peakCpuPercent": 1196.2, "firstAtMs": 0, "lastAtMs": 95057 }, "gpu": { "available": true, "sampleCount": 17, "peakUtilPercent": 99, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 27389, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 8534, "totalOutputTokens": 619, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4907, "todayTokensCached": 0, "todayTokensOut": 562, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4907, "totalTokensCached": 0, "totalTokensOut": 562, "totalCost": 0, "medianOutputTokensPerSec": 9.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.2, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 4907, "outputTokens": 562, "cost": 0, "durationMs": 89920, "outputTokensPerSec": 9.21570309153327, "at": "2026-11-09T21:38:45.084Z" }, "lastUpdated": "2026-11-09T21:38:45.084Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:38:48.458Z" } }, "source": "engine-log" }, "derived": { "durationMs": 97698, "meanTokensPerSec": 9.25, "genTokensPerSec": 9.25, "genTokens": 619, "promptTokensPerSec": 586.66, "peakRssMb": 9932, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 4907, "output": 562 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:37:10.756Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:37:10.642Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "priya": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.0513458251953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0513458251953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "durationMs": 106549, "timeoutMs": 2269289, "budgetUsedFraction": 0.046952591758916556 }, "timing": { "startedAt": "2026-11-09T21:37:05.191Z", "finishedAt": "2026-11-09T21:38:51.740Z", "timeToFirstArtifactMs": 99882, "timeToLastArtifactWriteMs": 99893, "timeToFirstTokenMs": 24510, "firstTurnTtftMs": 14537, "timeToFirstToolCallMs": 29918 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 3, "byTool": { "read_file": 2, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 10096, "filePath": "failing-tests-spec", "bytes": 402, "score": 2, "scoreMax": 4, "signals": [ "tests-untouched", "tsc-clean" ], "failReason": "vitest run failed (exit 1): × starts in \"draft\" 4ms | × starts with an empty history log 0ms | × submit moves draft to pending and returns the new state 0ms | × pay moves pending to paid 0ms | FAIL tests/machine.test.ts > order machine: initial state > starts in \"draft\" | Error: createOrderMachine is not implemented yet | FAIL tests/machine.test.ts > order machine: initial state > starts with an empty history log | FAIL tests/machine.test.ts > order machine: transitions > submit moves draft to pending and returns the new state" }, { "atMs": 102934, "filePath": "failing-tests-spec", "bytes": 1766, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } ], "latest": { "filePath": "failing-tests-spec", "bytes": 1766, "score": 4, "scoreMax": 4, "signals": [ "tests-untouched", "machine-implemented", "vitest-green", "tsc-clean" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T21:37:05.195Z] [trial] id=failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve scenario=failing-tests-spec model=underdog-saluki-27b-iq2-mix", "[2026-11-09T21:37:05.195Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T21:37:05.195Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-failing-tests-spec-tKE0Ud", "[2026-11-09T21:37:05.195Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "[2026-11-09T21:37:05.195Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T21:37:05.199Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T21:37:05.203Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T21:37:05.208Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-failing-tests-spec-tKE0Ud", "[2026-11-09T21:37:05.208Z] [trial] throughput-scaled ceiling: 25m → 38m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T21:37:10.704Z] [trial] daemon spawned pid=23896 port=https://127.0.0.1:63211", "[2026-11-09T21:37:11.596Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T21:37:11.613Z] [trial] meester=vanya", "[2026-11-09T21:37:11.613Z] [trial] running scenario setup hook", "[2026-11-09T21:37:12.068Z] [trial] scenario setup complete", "[2026-11-09T21:37:12.069Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T21:37:12.146Z] [poll] initial digests hard=1182ffe87b3afa9a soft=9ad846bb91c64312", "[2026-11-09T21:37:15.336Z] [poll] hard-progress: 1182ffe87b3afa9a → 0e3a3aad02b887a2 after 3s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=0,slot=0,stream=0) sniff=2/402B)", "[2026-11-09T21:37:35.817Z] [poll] hard-progress: 0e3a3aad02b887a2 → a7ce389a1b89897c after 20s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=18,stream=36) sniff=2/402B)", "[2026-11-09T21:38:48.125Z] [poll] terminal: success=true reason=vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)", "[2026-11-09T21:38:48.492Z] [capture] dumped 1 session(s)", "[2026-11-09T21:38:48.495Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-09T21:38:48.553Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-09T21:38:51.739Z] [trial] daemon shut down", "[2026-11-09T21:38:51.748Z] [trial] PASS duration=106549ms reason=vitest and strict tsc gates green with the seeded spec untouched (signals: tests-untouched, machine-implemented, vitest-green, tsc-clean)" ] }, "metrics": { "process": { "sampleCount": 17, "peakRssMb": 9932, "peakCpuPercent": 1196.2, "firstAtMs": 0, "lastAtMs": 95057 }, "gpu": { "available": true, "sampleCount": 17, "peakUtilPercent": 99, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 17, "peakUsedMb": 27389, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 8534, "totalOutputTokens": 619, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 4907, "todayTokensCached": 0, "todayTokensOut": 562, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 4907, "totalTokensCached": 0, "totalTokensOut": 562, "totalCost": 0, "medianOutputTokensPerSec": 9.2, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 9.2, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 4907, "outputTokens": 562, "cost": 0, "durationMs": 89920, "outputTokensPerSec": 9.21570309153327, "at": "2026-11-09T21:38:45.084Z" }, "lastUpdated": "2026-11-09T21:38:45.084Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:38:48.458Z" } }, "source": "engine-log" }, "derived": { "durationMs": 97698, "meanTokensPerSec": 9.25, "genTokensPerSec": 9.25, "genTokens": 619, "promptTokensPerSec": 586.66, "peakRssMb": 9932, "peakGpuUtilPercent": 99, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 4907, "output": 562 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T21:37:05.191Z", "finishedAt": "2026-11-09T21:38:51.740Z", "timeToFirstArtifactMs": 99882, "timeToLastArtifactWriteMs": 99893, "timeToFirstTokenMs": 24510, "firstTurnTtftMs": 14537, "timeToFirstToolCallMs": 29918 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:37:10.756Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:37:10.642Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "priya": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 2, "maxContextFill": 0.0513458251953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0513458251953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-failing-tests-spec", "baselineCellId": "r1-saluki-cuda-core-failing-tests-spec", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "scenarioId": "failing-tests-spec", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "reviewedAt": "2026-11-09T21:51:04.802Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/workspace/machine-kata/src/machine.ts", "artifactBytes": 1766, "artifactSha256": "e6c9a3e697039d21f5fccfd5bc15e5eb821018d44d3ffed265c83e27b6d73c1a", "axisSource": "evals/src/scenarios/failing-tests-spec.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The seeded specification stays unchanged and standard Vitest/strict tsc gates pass. Nominal transitions, guard and history are implemented concisely. A separate probe finds that inherited event toString is accepted, corrupts state to undefined and makes a later can call throw.", "reviewBasis": "Read complete primary artifacts within the 20KiB cap, all declared companion outputs and relevant scenario contract. Inspected tool receipts and terminal facts. Browser checks/images or production-export probes are saved separately; no artifact or fixed evidence was changed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/workspace/machine-kata/src/machine.ts", "bytes": 1766, "sha256": "e6c9a3e697039d21f5fccfd5bc15e5eb821018d44d3ffed265c83e27b6d73c1a", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 1766, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T21:37:13.127Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T21:37:16.505Z INFO [chat] [llama-server] launch {\"pid\":34920,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T21:38:51.019Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-34920-1794260331019\",\"pid\":34920,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":94514,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 5468, "peakGpuMiB": 10636, "peakProcessRssMiB": 9932, "peakSystemRamMiB": 27389, "decodeTokensPerSec": 9.25, "prefillTokensPerSec": 586.66, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0513458251953125 }, "note": "Hybrid task placement, 39/64 dense FFN layers on GPU and 25 CPU. Whole-system memory includes other applications; shared GPU memory and paging are unmeasured. Multiple launches alone do not imply a native crash." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:26.168Z", "reviewId": "091", "cellId": "r3-saluki-cuda-core-failing-tests-spec", "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "scenarioId": "failing-tests-spec", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "baselineCellId": "r1-saluki-cuda-core-failing-tests-spec", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "a8797968a687f1f10f5ed4a4b986b6d83ea7fb157d76ded2a46744e1d278eae4", "evidenceSha256": "a69bb9c2d981b2af5194e3f8ba26b8d9063f89f67eac6684e40990ff8ddc9c52", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/workspace/machine-kata/src/machine.ts", "bytes": 1766, "sha256": "e6c9a3e697039d21f5fccfd5bc15e5eb821018d44d3ffed265c83e27b6d73c1a", "shownBytes": 1766, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes are declared. The src/machine.ts shown is a coherent, strictly typed state machine with states, a guarded expedite transition, a history log and a formatted error message. The tests/ spec is not in the evidence, so I can't confirm the inferred behavior or the exact error text.", "limitations": "The test file and the harness results (vitest and tsc) were not shown. The run ended as runner-terminal, so I can't say whether all tests passed. Whether tests/ was left unmodified can't be verified. This is a source-only review with no execution.", "meanScore": null, "durationMs": 14401, "costUsd": 0.01262, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2394, "cache_read_input_tokens": 700, "output_tokens": 297, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2394, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 297, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2394, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2394 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 297, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2394, "webSearchRequests": 0, "costUSD": 0.01262, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2394, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - failing-tests-spec / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 1m 47s; budget used: 4.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 99882`; `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1766,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.046952591758916556`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 99882`; `facts.outcome.durationMs: 106549` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9932 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 99% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10636 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27389 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 9.25 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 8534 + 619 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 5% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 99882`\n- `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1766,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.046952591758916556`\n- `facts.toolUse.totalToolCalls: 3`\n- `facts.timing.timeToFirstArtifactMs: 99882`\n- `facts.outcome.durationMs: 106549`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 99882`; `facts.sniff.latest: {\"filePath\":\"failing-tests-spec\",\"bytes\":1766,\"score\":4,\"scoreMax\":4,\"signals\":[\"tests-untouched\",\"machine-implemented\",\"vitest-green\",\"tsc-clean\"],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.046952591758916556`; `facts.toolUse.totalToolCalls: 3`; `facts.timing.timeToFirstArtifactMs: 99882`; `facts.outcome.durationMs: 106549`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed composite **10/10**, pass; eligibility unchanged. The advisory status is **notApplicable** (no declared axes). No declared axes. The seeded specification stays unchanged and standard Vitest/strict tsc gates pass. Nominal transitions, guard and history are implemented concisely. A separate probe finds that inherited event toString is accepted, corrupts state to undefined and makes a later can call throw.\n\n\n\n## Positives\n\nThe original specification and strict typecheck pass after one implementation write and two reads (toolUse.totalToolCalls=3; sniff.latest signals tests-untouched, machine-implemented, vitest-green, tsc-clean). Normal transitions, guard and ordered history are explicit in the table.\n\n## Negatives\n\nTRANSITIONS[state][event] does not check own properties. The separate production-export probe can(toString)=true, send(toString) returns undefined and corrupts state, then can(submit) throws. This contradicts the broad unknown-event/no-throw intent beyond the literal seeded event list (advisory-machine-checks.json). The same defect occurred in prior rounds; fixed 10 remains unchanged.\n\n## Performance and fit\n\nDecode 9.25 tok/s; prefill 586.66 tok/s. Peak GPU 10636/12227 MiB, process-tree RSS 9932 MiB, system RAM 27389/31865 MiB (facts.perf). Context 65536, 1 slot, f16; max observed released slot 5468 tokens. Explicit native/OOM/context fault lines: 0. Hybrid placement is not full GPU residency, and shared GPU memory/paging remain unmeasured. Petshop includes image-engine use; a second chat launch alone is not classified as a crash.\n\n## Tactical experiment — not applied\n\nExperiment only: add a generic own-property boundary check for string-keyed dispatch tables to the reusable coding checklist in ../gilde/data/gezel-templates/bu/builder/versions/1.5.1/about.md. Validate against unrelated route maps and parsers, not the fixture event list.\n\n## Strategic experiment — not applied\n\nExperiment only: surface optional prototype-inherited dispatch-key diagnostics in packages/mcp/src/validate.ts, with a versioned adversarial probe set. This should protect any user-supplied map key, without changing frozen tests or existing scores.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/facts.json): pass, fixed 10, 1.77 min, eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/facts.json): pass, fixed 9.5, 19.05 min, eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/facts.json): fail, fixed 6.1, 2.28 min, eligible.\n\nUnrelated controls include original R2 codemod sweep (pass 8.5) and constrained communications (fail 6). A successful artifact or compile gate does not establish broad workflow or output quality. The proposals target reusable validation, state boundaries and tool recovery, with no fixture-specific global hint. Original/continuation GEZEL_MXC_EXEC is an explicit comparison boundary; other-model/native history and differing automatic timeout scaling prevent causal speed/quality attribution.\n\n## Recommended next experiment\n\nEvaluate the proposed diagnostic on one unrelated task type and record advisory defects, repair turns and false positives without modifying this campaign's fixed evidence.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, historical-comparison.json, plus advisory-ui-checks-20261109 or advisory production-export checks where present.\n" }, { "cellId": "r3-saluki-cuda-core-symptom-debug", "baselineCellId": "r1-saluki-cuda-core-symptom-debug", "suite": "core", "scenario": "symptom-debug", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T21:41:18.852Z", "finishedAt": "2026-11-09T21:48:57.981Z", "durationMs": 459129, "success": true, "reason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 450124 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "symptom-debug", "bytes": 1209, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.25290358047207523 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 450124 }, { "source": "facts", "path": "outcome.durationMs", "value": 459129 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "scenarioId": "symptom-debug", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 76, "peakRssMb": 9480, "peakCpuPercent": 1205, "firstAtMs": 1, "lastAtMs": 447646 }, "gpu": { "available": true, "sampleCount": 76, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 76, "peakUsedMb": 26808, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 5431, "totalOutputTokens": 3628, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:48:53.924Z" } }, "source": "engine-log" }, "derived": { "durationMs": 449488, "meanTokensPerSec": 8.57, "genTokensPerSec": 8.57, "genTokens": 3628, "promptTokensPerSec": 591.5, "peakRssMb": 9480, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:41:24.487Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:41:24.386Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "theo": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0552825927734375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0552825927734375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "durationMs": 459129, "timeoutMs": 1815431, "budgetUsedFraction": 0.25290358047207523 }, "timing": { "startedAt": "2026-11-09T21:41:18.852Z", "finishedAt": "2026-11-09T21:48:57.981Z", "timeToFirstArtifactMs": 450124, "timeToLastArtifactWriteMs": 450124, "timeToFirstTokenMs": 23689, "firstTurnTtftMs": 14133, "timeToFirstToolCallMs": 34849 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 4, "byTool": { "read_file": 3, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 7125, "filePath": "symptom-debug", "bytes": 1193, "score": 1, "scoreMax": 3, "signals": [ "accept-untouched" ], "failReason": "node accept.mjs still failing: CASE 3: expected \"a4\" got \"a5\" | CASE 4: expected [\"a1\",\"a2\",\"a3\",\"a4\",\"a5\",\"a6\",\"a7\",\"a8\",\"a9\",\"a10\"] got [\"a1\",\"a2\",\"a3\",\"a5\",\"a6\",\"a7\",\"a9\",\"a10\"] | CASE 6: expected [\"a1\",\"a2\",\"a3\",\"a4\",\"a5\",\"a6\",\"a7\",\"a8\",\"a9\",\"a10\"] got [\"a1\",\"a2\",\"a3\",\"a4\",\"a6\",\"a7\",\"a8\",\"a9\"]" }, { "atMs": 454593, "filePath": "symptom-debug", "bytes": 1209, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } ], "latest": { "filePath": "symptom-debug", "bytes": 1209, "score": 3, "scoreMax": 3, "signals": [ "accept-untouched", "paginate-changed", "accept-passes" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T21:41:18.855Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-symptom-debug-HDdCcK", "[2026-11-09T21:41:18.855Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T21:41:18.856Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T21:41:18.855Z] [trial] id=symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn scenario=symptom-debug model=underdog-saluki-27b-iq2-mix", "[2026-11-09T21:41:18.855Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "[2026-11-09T21:41:18.859Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T21:41:18.862Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T21:41:18.868Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-symptom-debug-HDdCcK", "[2026-11-09T21:41:18.868Z] [trial] throughput-scaled ceiling: 20m → 30m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T21:41:24.385Z] [trial] daemon spawned pid=33484 port=https://127.0.0.1:53671", "[2026-11-09T21:41:25.306Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T21:41:25.321Z] [trial] meester=cassian", "[2026-11-09T21:41:25.321Z] [trial] running scenario setup hook", "[2026-11-09T21:41:25.719Z] [trial] scenario setup complete", "[2026-11-09T21:41:25.721Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T21:41:25.800Z] [poll] initial digests hard=cf52730005ccfee5 soft=ea634c6a5b9cd875", "[2026-11-09T21:41:26.093Z] [poll] hard-progress: cf52730005ccfee5 → cab1f129aca1a55f after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/1193B)", "[2026-11-09T21:41:55.128Z] [poll] hard-progress: cab1f129aca1a55f → 7404223b9e8739d7 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=30,stream=68) sniff=1/1193B)", "[2026-11-09T21:48:53.445Z] [poll] terminal: success=true reason=node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)", "[2026-11-09T21:48:53.962Z] [capture] dumped 1 session(s)", "[2026-11-09T21:48:53.966Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-09T21:48:54.041Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-09T21:48:57.979Z] [trial] daemon shut down", "[2026-11-09T21:48:57.987Z] [trial] PASS duration=459129ms reason=node accept.mjs prints ALL CASES PASS with accept.mjs untouched (signals: accept-untouched, paginate-changed, accept-passes)" ] }, "metrics": { "process": { "sampleCount": 76, "peakRssMb": 9480, "peakCpuPercent": 1205, "firstAtMs": 1, "lastAtMs": 447646 }, "gpu": { "available": true, "sampleCount": 76, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 76, "peakUsedMb": 26808, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 5431, "totalOutputTokens": 3628, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:48:53.924Z" } }, "source": "engine-log" }, "derived": { "durationMs": 449488, "meanTokensPerSec": 8.57, "genTokensPerSec": 8.57, "genTokens": 3628, "promptTokensPerSec": 591.5, "peakRssMb": 9480, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T21:41:18.852Z", "finishedAt": "2026-11-09T21:48:57.981Z", "timeToFirstArtifactMs": 450124, "timeToLastArtifactWriteMs": 450124, "timeToFirstTokenMs": 23689, "firstTurnTtftMs": 14133, "timeToFirstToolCallMs": 34849 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:41:24.487Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:41:24.386Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "theo": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0552825927734375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0552825927734375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-symptom-debug", "baselineCellId": "r1-saluki-cuda-core-symptom-debug", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "scenarioId": "symptom-debug", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "reviewedAt": "2026-11-09T21:54:37.842Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/workspace/pagination-service/lib/paginate.mjs", "artifactBytes": 1209, "artifactSha256": "626a0f50583cc62bec0d3048b839f084b86cb983c10a071d829595cef942ee08", "axisSource": "evals/src/scenarios/symptom-debug.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The minimal patch removes the extra cursor increment and updates its misleading comment. The acceptance script stays unchanged; the runner passes its cases plus a hidden traversal/empty-input/invalid-limit probe. Four tool calls suffice, though the small patch takes 7.65 minutes.", "reviewBasis": "Read all 1209 bytes of the declared pagination module, the unchanged acceptance script, relevant scenario gates and all four tool receipts. No additional execution or source changes were needed for the one-line repair.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/workspace/pagination-service/lib/paginate.mjs", "bytes": 1209, "sha256": "626a0f50583cc62bec0d3048b839f084b86cb983c10a071d829595cef942ee08", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 1209, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T21:41:26.459Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T21:41:29.700Z INFO [chat] [llama-server] launch {\"pid\":24780,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T21:48:57.262Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-24780-1794260937262\",\"pid\":24780,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":447562,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 8948, "peakGpuMiB": 10636, "peakProcessRssMiB": 9480, "peakSystemRamMiB": 26808, "decodeTokensPerSec": 8.57, "prefillTokensPerSec": 591.5, "note": "Task uses hybrid placement: 39/64 dense FFN layers GPU, 25 CPU. Shared GPU memory and paging remain unmeasured; system memory includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:28.092Z", "reviewId": "092", "cellId": "r3-saluki-cuda-core-symptom-debug", "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "scenarioId": "symptom-debug", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "baselineCellId": "r1-saluki-cuda-core-symptom-debug", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "c91fed434df632eb751a4d8ca3f6be968d63168bb006f9f84d59fc51bed30099", "evidenceSha256": "ce0558deb5fddb7bf2c58cf225f102dccf6dfc8740803ce397ac0d113627af0d", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/workspace/pagination-service/lib/paginate.mjs", "bytes": 1209, "sha256": "626a0f50583cc62bec0d3048b839f084b86cb983c10a071d829595cef942ee08", "shownBytes": 1209, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes are declared. The only artifact, paginate.mjs, looks internally consistent: the cursor encodes the next start index and nextCursor is null at the end. Without accept.mjs or its output, I can't tell whether it fixes the failing cases or was edited at all.", "limitations": "The evidence has no accept.mjs output, no original file for comparison, no edit history and no proof that accept.mjs is unchanged. The run's terminal state is unclear, so I make no success or failure claim.", "meanScore": null, "durationMs": 14095, "costUsd": 0.011392, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2127, "cache_read_input_tokens": 700, "output_tokens": 281, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2127, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 281, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2127, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2127 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 281, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2127, "webSearchRequests": 0, "costUSD": 0.011392, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2127, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - symptom-debug / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 7m 39s; budget used: 25.3%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 450124`; `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1209,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.25290358047207523`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: 450124`; `facts.outcome.durationMs: 459129` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9480 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10636 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26808 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.57 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 5431 + 3628 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 450124`\n- `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1209,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.25290358047207523`\n- `facts.toolUse.totalToolCalls: 4`\n- `facts.timing.timeToFirstArtifactMs: 450124`\n- `facts.outcome.durationMs: 459129`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 450124`; `facts.sniff.latest: {\"filePath\":\"symptom-debug\",\"bytes\":1209,\"score\":3,\"scoreMax\":3,\"signals\":[\"accept-untouched\",\"paginate-changed\",\"accept-passes\"],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.25290358047207523`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: 450124`; `facts.outcome.durationMs: 459129`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **10/10 pass**, unchanged. Advisory **notApplicable**: no declared axes. No declared axes. The minimal patch removes the extra cursor increment and updates its misleading comment. The acceptance script stays unchanged; the runner passes its cases plus a hidden traversal/empty-input/invalid-limit probe. Four tool calls suffice, though the small patch takes 7.65 minutes.\n\n## Positives\n\nThe agent read acceptance, implementation and package metadata, then made one two-line replacement (one comment and one executable line). It aligned cursor consumption with the next-item encoding without changing the acceptance script. All three fixed signals pass, including the harness-owned hidden traversal, empty-input and invalid-limit checks (facts.sniff.latest; scenario runAcceptInDir). There are no red flags, interventions, repeat aborts or compactions.\n\n## Negatives and limits\n\nThe small edit still required 459129 ms (7.65 minutes), with 3628 generated tokens at 8.57 decode tok/s. Tool receipts run from the three initial reads at 21:41:53Z to the only edit at 21:48:48Z. The model did not itself execute acceptance before the runner stopped it; the runner supplied the observed validation. No new correctness defect was found in this focused repair. This narrow success does not demonstrate independent multi-stage debugging or broad task competence.\n\n## Performance and fit\n\nDecode 8.57 tok/s; prefill 591.5 tok/s. Peak GPU 10636/12227 MiB; process RSS 9480 MiB; system RAM 26808/31865 MiB. Context 65536, 1 slot, f16, maximum observed released slot 8948. Explicit native/OOM/context fault lines: 0. Actual task placement remains 39 dense FFN GPU layers and 25 CPU layers; shared GPU memory and paging are unmeasured.\n\n## Tactical experiment — not applied\n\nDo not change the repair guidance based on this passing result. If latency becomes the priority, test a generic concise-diagnosis instruction in the reusable coding-role about.md in Gilde, measuring generated tokens and wall time while holding sampling and fixed gates constant. The concrete target is the 415-second read-to-edit gap for one executable-line change, not a pagination-specific hint. Locate the role actually resolved by provisionScenarioGezel before proposing a content edit.\n\n## Strategic experiment — not applied\n\nIn packages/mcp/src/validate.ts, consider exposing an explicit known local acceptance command/result as reusable validation evidence. Evaluate whether this improves model-owned verification without redundant executions when the harness already validates. No framework defect or change is established by this passing trial.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/facts.json): pass, fixed 10, 3.92 min; eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/facts.json): pass, fixed 10, 7.72 min; eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/facts.json): pass, fixed 10, 0.59 min; eligible.\n\nUnrelated original R2 codemod sweep passed at 8.5 while constrained communications failed at 6. Compare any concise-diagnosis experiment against an unrelated schema migration and a multi-file refactor, and track hidden-check regressions as well as latency. Original/continuation comparisons cross the GEZEL_MXC_EXEC boundary; other-model history and automatic preflight timing prevent causal attribution.\n\n## Recommended next experiment\n\nAfter this frozen campaign, run the proposed concise-diagnosis comparison on several unrelated semantic repairs and multi-file changes. Preserve correctness gates and report generation cost separately from task success. No implementation, prompt, tuning or gate was changed here.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-core-data-wrangle", "baselineCellId": "r1-saluki-cuda-core-data-wrangle", "suite": "core", "scenario": "data-wrangle", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T21:51:35.161Z", "finishedAt": "2026-11-09T21:56:10.182Z", "durationMs": 275021, "success": true, "reason": "out/customers.json passed all 11 property checks (parses-as-array, row-shape, emails-normalized, dates-iso, unique-emails, source-ids-preserved, row-count, sorted-by-email, golden-bom-row, golden-quoted-name, golden-dedup-newest)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 225579 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "data-wrangle", "bytes": 6986, "score": 11, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count", "sorted-by-email", "golden-bom-row", "golden-quoted-name", "golden-dedup-newest" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.06732921718012495 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 225579 }, { "source": "facts", "path": "outcome.durationMs", "value": 275021 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "scenarioId": "data-wrangle", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 45, "peakRssMb": 10410, "peakCpuPercent": 1197.4, "firstAtMs": 0, "lastAtMs": 261878 }, "gpu": { "available": true, "sampleCount": 45, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 45, "peakUsedMb": 27202, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 8688, "totalOutputTokens": 1849, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:56:04.346Z" } }, "source": "engine-log" }, "derived": { "durationMs": 263427, "meanTokensPerSec": 8.54, "genTokensPerSec": 8.54, "genTokens": 1849, "promptTokensPerSec": 346.57, "peakRssMb": 10410, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:51:40.968Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:51:40.864Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.059417724609375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.059417724609375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "out/customers.json passed all 11 property checks (parses-as-array, row-shape, emails-normalized, dates-iso, unique-emails, source-ids-preserved, row-count, sorted-by-email, golden-bom-row, golden-quoted-name, golden-dedup-newest)", "durationMs": 275021, "timeoutMs": 4084720, "budgetUsedFraction": 0.06732921718012495 }, "timing": { "startedAt": "2026-11-09T21:51:35.161Z", "finishedAt": "2026-11-09T21:56:10.182Z", "timeToFirstArtifactMs": 225579, "timeToLastArtifactWriteMs": 225579, "timeToFirstTokenMs": 28361, "firstTurnTtftMs": 18670, "timeToFirstToolCallMs": 43553 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "list_dir": 1, "read_file": 5, "write_file": 1, "stat": 1, "run_nodejs_script": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 10 }, "sniff": { "progression": [ { "atMs": 7148, "filePath": "data-wrangle", "bytes": 0, "score": 0, "scoreMax": null, "signals": [], "failReason": null }, { "atMs": 268672, "filePath": "data-wrangle", "bytes": 6986, "score": 11, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count", "sorted-by-email", "golden-bom-row", "golden-quoted-name", "golden-dedup-newest" ], "failReason": null } ], "latest": { "filePath": "data-wrangle", "bytes": 6986, "score": 11, "scoreMax": 11, "signals": [ "parses-as-array", "row-shape", "emails-normalized", "dates-iso", "unique-emails", "source-ids-preserved", "row-count", "sorted-by-email", "golden-bom-row", "golden-quoted-name", "golden-dedup-newest" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T21:51:35.165Z] [trial] id=data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0 scenario=data-wrangle model=underdog-saluki-27b-iq2-mix", "[2026-11-09T21:51:35.165Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-data-wrangle-g5UTwg", "[2026-11-09T21:51:35.165Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "[2026-11-09T21:51:35.165Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T21:51:35.165Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T21:51:35.168Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T21:51:35.171Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T21:51:35.176Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-data-wrangle-g5UTwg", "[2026-11-09T21:51:35.176Z] [trial] throughput-scaled ceiling: 45m → 68m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T21:51:40.860Z] [trial] daemon spawned pid=16628 port=https://127.0.0.1:61056", "[2026-11-09T21:51:41.786Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T21:51:41.803Z] [trial] meester=wren", "[2026-11-09T21:51:41.803Z] [trial] running scenario setup hook", "[2026-11-09T21:51:42.226Z] [trial] scenario setup complete", "[2026-11-09T21:51:42.227Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T21:51:42.298Z] [poll] initial digests hard=df001ccab3907ec4 soft=72d04bf9fb77bb9e", "[2026-11-09T21:51:42.364Z] [poll] hard-progress: df001ccab3907ec4 → 71fd838f938c05ba after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-09T21:52:23.355Z] [poll] hard-progress: 71fd838f938c05ba → ed976e02ec6fd97c after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=36,stream=85) sniff=0/0B)", "[2026-11-09T21:52:35.298Z] [poll] hard-progress: ed976e02ec6fd97c → e0a9b5e57fff3835 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=55,stream=137) sniff=0/0B)", "[2026-11-09T21:55:22.008Z] [poll] hard-progress: e0a9b5e57fff3835 → ca893c74748bff8d after 167s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=511,stream=2350) sniff=0/0B)", "[2026-11-09T21:55:34.016Z] [poll] hard-progress: ca893c74748bff8d → ee4591add7104578 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=537,stream=2430) sniff=0/0B)", "[2026-11-09T21:55:39.855Z] [poll] hard-progress: ee4591add7104578 → 857586ee9d02256b after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=547,stream=2461) sniff=0/0B)", "[2026-11-09T21:55:57.921Z] [poll] hard-progress: 857586ee9d02256b → 974a2ee87391336f after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=582,stream=2565) sniff=0/0B)", "[2026-11-09T21:56:03.834Z] [poll] terminal: success=true reason=out/customers.json passed all 11 property checks (parses-as-array, row-shape, emails-normalized, dates-iso, unique-emails, source-ids-preserved, row-count, sorted-by-email, golden-bom-row, golden-quoted-name, golden-dedup-newest)", "[2026-11-09T21:56:04.400Z] [capture] dumped 1 session(s)" ] }, "metrics": { "process": { "sampleCount": 45, "peakRssMb": 10410, "peakCpuPercent": 1197.4, "firstAtMs": 0, "lastAtMs": 261878 }, "gpu": { "available": true, "sampleCount": 45, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 45, "peakUsedMb": 27202, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 8688, "totalOutputTokens": 1849, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T21:56:04.346Z" } }, "source": "engine-log" }, "derived": { "durationMs": 263427, "meanTokensPerSec": 8.54, "genTokensPerSec": 8.54, "genTokens": 1849, "promptTokensPerSec": 346.57, "peakRssMb": 10410, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T21:51:35.161Z", "finishedAt": "2026-11-09T21:56:10.182Z", "timeToFirstArtifactMs": 225579, "timeToLastArtifactWriteMs": 225579, "timeToFirstTokenMs": 28361, "firstTurnTtftMs": 18670, "timeToFirstToolCallMs": 43553 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:51:40.968Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:51:40.864Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.059417724609375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.059417724609375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-data-wrangle", "baselineCellId": "r1-saluki-cuda-core-data-wrangle", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "scenarioId": "data-wrangle", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "reviewedAt": "2026-11-09T22:25:47.628Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/out/customers.json", "artifactBytes": 6986, "artifactSha256": "94e094e171033476d9560d54b651ac8fe1537d56e13d1695f283714925ca930a", "axisSource": "evals/src/scenarios/data-wrangle.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The output contains all 56 correctly normalized, deduplicated and sorted records. Independent CSV/date parsing found zero field differences against source rows. The reusable helper handles the supplied quoted commas and BOM, but is not a general CSV parser and does not reject impossible calendar dates.", "reviewBasis": "Read complete declared outputs within 20KiB per primary artifact, relevant sources/contracts and tool receipts. Supplemental checks are advisory; no source, artifact, fixed evidence or gate changed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/out/customers.json", "bytes": 6986, "sha256": "94e094e171033476d9560d54b651ac8fe1537d56e13d1695f283714925ca930a", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 6986, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/tools/clean_customers.mjs", "bytes": 3066, "sha256": "170b3caf0c4657e29d81c83f83ef53059eb721bdef7005ad707b1ffa59f5f783", "role": "producedCompanion", "status": "reviewed", "reviewedBytes": 3066, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T21:51:42.954Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T21:51:46.170Z INFO [chat] [llama-server] launch {\"pid\":54112,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T21:56:09.480Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-54112-1794261369480\",\"pid\":54112,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":263311,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 8825, "peakGpuMiB": 10636, "peakProcessRssMiB": 10410, "peakSystemRamMiB": 27202, "decodeTokensPerSec": 8.54, "prefillTokensPerSec": 346.57, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.059417724609375 }, "note": "Actual task placement is hybrid: 39/64 dense FFN layers on GPU and 25 CPU. Shared GPU memory and paging are unmeasured. Whole-system RAM includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:34.795Z", "reviewId": "093", "cellId": "r3-saluki-cuda-core-data-wrangle", "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "scenarioId": "data-wrangle", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "baselineCellId": "r1-saluki-cuda-core-data-wrangle", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "4572781813a0fcbb38fbfb3870194c58fa78848c3853c334e61df9e1ccd218ca", "evidenceSha256": "b540c7cb170d7d87f198a63167b19fea2055e6548439cb3cde1f32dae65a2fba", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/out/customers.json", "bytes": 6986, "sha256": "94e094e171033476d9560d54b651ac8fe1537d56e13d1695f283714925ca930a", "shownBytes": 6986, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/tools/clean_customers.mjs", "bytes": 3066, "sha256": "170b3caf0c4657e29d81c83f83ef53059eb721bdef7005ad707b1ffa59f5f783", "shownBytes": 3066, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0\\workspace\\customer-data-cleanup\\data\\raw\\customers_a.csv", "bytes": 1045, "sha256": "bc92617373571bdf0bef1c29ee05aad44a7a68274c0f7334b1ebe8aa5eedfe54", "shownBytes": 1045, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0\\workspace\\customer-data-cleanup\\data\\raw\\customers_b.csv", "bytes": 1087, "sha256": "ff167222077dd5a971ee88d6ab656d98b2b2051e84ceb223bcc893aab9186b09", "shownBytes": 1087, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0\\workspace\\customer-data-cleanup\\data\\raw\\legacy_export.csv", "bytes": 1148, "sha256": "90e6abcfcf404de96b3ac6385e3ddc3265125552377f3c3428d5636d310f9afa", "shownBytes": 1148, "truncated": false } ], "scoreAxes": {}, "justification": "No numeric axes are declared. Qualitatively, out/customers.json looks correct: 56 records, which is 60 rows minus 4 duplicate emails (alice, omar, zoe, robert). The newest row was kept for each. Dates are ISO, emails are lowercased and trimmed, quoted comma names are intact, and the output is sorted. The script handles the BOM and the banner line.", "limitations": "Source-only review. I did not run the script or tools/check_customers.mjs, so I can't confirm the checker passed. The sort uses localeCompare, which could differ from a byte-order sort in edge cases. The evidence doesn't show whether the file has a BOM.", "meanScore": null, "durationMs": 18008, "costUsd": 0.054244, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 10820, "cache_read_input_tokens": 700, "output_tokens": 1089, "output_tokens_details": { "thinking_tokens": 750 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 10820, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 1089, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 10820, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 10820 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 1089, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 10820, "webSearchRequests": 0, "costUSD": 0.054244, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 750, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 10820, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - data-wrangle / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 4m 35s; budget used: 6.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 225579`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":11,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.06732921718012495`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 225579`; `facts.outcome.durationMs: 275021` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10410 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10636 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27202 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.54 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 8688 + 1849 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 225579`\n- `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":11,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.06732921718012495`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 225579`\n- `facts.outcome.durationMs: 275021`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 225579`; `facts.sniff.latest: {\"filePath\":\"data-wrangle\",\"bytes\":6986,\"score\":11,\"scoreMax\":11,\"signals\":[\"parses-as-array\",\"row-shape\",\"emails-normalized\",\"dates-iso\",\"unique-emails\",\"source-ids-preserved\",...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.06732921718012495`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 225579`; `facts.outcome.durationMs: 275021`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed composite **10/10**, pass; eligibility unchanged. Advisory **notApplicable** because the scenario declares no judge axes. No declared axes. The output contains all 56 correctly normalized, deduplicated and sorted records. Independent CSV/date parsing found zero field differences against source rows. The reusable helper handles the supplied quoted commas and BOM, but is not a general CSV parser and does not reject impossible calendar dates.\n\n## Positives\n\nAll 11 fixed checks pass (facts.sniff.latest.score=11/11), and a separate read-only PowerShell ConvertFrom-Csv/ParseExact comparison confirms every id, name, email and date in all 56 records. The emitted output keeps source ids and the newest duplicate, including the quoted names. One script write and one actual script execution produce the deliverable in 275021 ms, with no red flags (facts.toolUse: 9 calls; byTool.run_nodejs_script=1).\n\n## Negatives and limits\n\nThe final dataset is correct for the supplied inputs. Source review of tools/clean_customers.mjs shows that its handwritten parser splits on physical newlines and drops escaped double quotes; Date construction normalizes impossible dates rather than rejecting them. Those are limits of reuse, not failures of this requested dataset. There are two extra post-write inspections (stat and a full helper read) before execution, visible in tool receipts and facts.toolUse.byTool. The model did not run the provided check_customers command before the automatic gate stopped the trial.\n\n## Performance and fit\n\nDecode 8.54 tok/s; prefill 346.57 tok/s. Peak GPU 10636/12227 MiB; process RSS 10410 MiB; whole-system RAM 27202/31865 MiB. Context 65536, 1 slot, f16; maximum observed released slot 8825. Explicit native/OOM/context fault lines: 0. Actual task placement is hybrid, with 39 dense FFN GPU layers and 25 CPU layers. Shared GPU memory and paging remain unmeasured. The brief preflight probe used different automatic settings and is not the task placement/performance measurement.\n\n## Tactical experiment — not applied\n\nExperiment only: in ../gilde/data/gezel-templates/de/developer/versions/1.5.1/about.md, add under Test what you change: \"After a successful script write, execute the script and its available output check; reread the source only when a write or runtime diagnostic requires it.\" Compare edit-to-execution latency and output correctness across unrelated data conversion and migration tasks. Evidence: facts.toolUse.byTool.stat=1, read_file=5, plus the post-write stat/read receipts. Do not remove checks or alter this campaign.\n\n## Strategic experiment — not applied\n\nExperiment only: enrich the write_file success receipt in packages/mcp/src/server.ts with byte count and content identity so the caller can distinguish a committed write from a failed or partial write without a separate stat/read cycle. Measure duplicate read turns and verify no reduction in actual output validation. For reusable ingestion work, expose parse diagnostics in packages/mcp/src/validate.ts rather than teaching fixture-specific CSV rules.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/facts.json): fail, fixed 4.3, 105.56 min, eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/facts.json): fail, fixed 3, 56.38 min, eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/facts.json): fail, fixed 3.4, 9.26 min, eligible.\n\nUnrelated original R2 controls are codemod sweep (pass 8.5) and constrained communications (fail 6). These checks test general evidence discipline, execution and audit completeness rather than scenario-specific expected strings. This campaign contains no prompt, tuning or framework treatment. Original and continuation fixed baselines remain separate at the explicit GEZEL_MXC_EXEC boundary; other-model history and automatic preflight timing differences prevent causal attribution.\n\n## Recommended next experiment\n\nAfter this frozen campaign, test the concrete tactical instruction on the unrelated task types named above, measuring output correctness, advisory defects and repair turns before calling it an improvement.\n\n## Strategic note\n\nThe original first repetition produced no final dataset; the second had correct records but failed ordering. This third repetition now passes both standard ordering and the independent full-record comparison. It is evidence of variability and successful execution, not proof that the environment change caused improvement.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-core-incident-postmortem", "baselineCellId": "r1-saluki-cuda-core-incident-postmortem", "suite": "core", "scenario": "incident-postmortem", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T21:58:34.963Z", "finishedAt": "2026-11-09T22:14:01.683Z", "durationMs": 926720, "success": true, "reason": "all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 918347 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "incident-postmortem", "bytes": 9446, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.2552341565171026 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 918347 }, { "source": "facts", "path": "outcome.durationMs", "value": 926720 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "scenarioId": "incident-postmortem", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 154, "peakRssMb": 9522, "peakCpuPercent": 1216, "firstAtMs": 1, "lastAtMs": 914396 }, "gpu": { "available": true, "sampleCount": 154, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 154, "peakUsedMb": 26849, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 15939, "totalOutputTokens": 7300, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T22:13:56.368Z" } }, "source": "engine-log" }, "derived": { "durationMs": 915989, "meanTokensPerSec": 8.36, "genTokensPerSec": 8.36, "genTokens": 7300, "promptTokensPerSec": 583.54, "peakRssMb": 9522, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:58:40.349Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:58:40.258Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "jordan": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.14569091796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.14569091796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "durationMs": 926720, "timeoutMs": 3630862, "budgetUsedFraction": 0.2552341565171026 }, "timing": { "startedAt": "2026-11-09T21:58:34.963Z", "finishedAt": "2026-11-09T22:14:01.683Z", "timeToFirstArtifactMs": 918347, "timeToLastArtifactWriteMs": 918347, "timeToFirstTokenMs": 34479, "firstTurnTtftMs": 25205, "timeToFirstToolCallMs": 50077 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "read_files": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 920895, "filePath": "incident-postmortem", "bytes": 9446, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } ], "latest": { "filePath": "incident-postmortem", "bytes": 9446, "score": 9, "scoreMax": 9, "signals": [ "file-present", "all-sections", "summary-concise", "evidence-citations", "timestamp-citations", "grounded-core-facts", "no-unsupported-certainty", "action-items-formatted", "no-blame-language" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T21:58:34.966Z] [trial] id=incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj scenario=incident-postmortem model=underdog-saluki-27b-iq2-mix", "[2026-11-09T21:58:34.967Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T21:58:34.967Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T21:58:34.967Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-incident-postmortem-l8TmBm", "[2026-11-09T21:58:34.967Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "[2026-11-09T21:58:34.969Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T21:58:34.972Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T21:58:34.978Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-incident-postmortem-l8TmBm", "[2026-11-09T21:58:34.979Z] [trial] throughput-scaled ceiling: 40m → 61m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T21:58:40.344Z] [trial] daemon spawned pid=33484 port=https://127.0.0.1:62173", "[2026-11-09T21:58:41.221Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T21:58:41.236Z] [trial] running scenario setup hook", "[2026-11-09T21:58:41.236Z] [trial] meester=yusuf", "[2026-11-09T21:58:41.634Z] [trial] scenario setup complete", "[2026-11-09T21:58:41.636Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T21:58:41.716Z] [poll] initial digests hard=2802fa639a26e94d soft=e3313556f6ce25e8", "[2026-11-09T21:58:41.800Z] [poll] hard-progress: 2802fa639a26e94d → dabb3ad0e3ffee70 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-09T21:59:28.905Z] [poll] hard-progress: dabb3ad0e3ffee70 → 79e529d4c04e3b7d after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=51,stream=169) sniff=0/0B)", "[2026-11-09T22:13:55.858Z] [poll] terminal: success=true reason=all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)", "[2026-11-09T22:13:56.418Z] [capture] dumped 1 session(s)", "[2026-11-09T22:13:56.422Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-09T22:13:56.504Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-09T22:14:01.682Z] [trial] daemon shut down", "[2026-11-09T22:14:01.690Z] [trial] PASS duration=926720ms reason=all hard signals firing (8/8; advisory no-blame-language=pass; signals: file-present, all-sections, summary-concise, evidence-citations, timestamp-citations, grounded-core-facts, no-unsupported-certainty, action-items-formatted, no-blame-language)" ] }, "metrics": { "process": { "sampleCount": 154, "peakRssMb": 9522, "peakCpuPercent": 1216, "firstAtMs": 1, "lastAtMs": 914396 }, "gpu": { "available": true, "sampleCount": 154, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 154, "peakUsedMb": 26849, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 15939, "totalOutputTokens": 7300, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T22:13:56.368Z" } }, "source": "engine-log" }, "derived": { "durationMs": 915989, "meanTokensPerSec": 8.36, "genTokensPerSec": 8.36, "genTokens": 7300, "promptTokensPerSec": 583.54, "peakRssMb": 9522, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T21:58:34.963Z", "finishedAt": "2026-11-09T22:14:01.683Z", "timeToFirstArtifactMs": 918347, "timeToLastArtifactWriteMs": 918347, "timeToFirstTokenMs": 34479, "firstTurnTtftMs": 25205, "timeToFirstToolCallMs": 50077 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T21:58:40.349Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T21:58:40.258Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "jordan": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.14569091796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.14569091796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-incident-postmortem", "baselineCellId": "r1-saluki-cuda-core-incident-postmortem", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "scenarioId": "incident-postmortem", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "reviewedAt": "2026-11-09T22:25:47.710Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/workspace/checkout-incident-postmortem/postmortem.md", "artifactBytes": 9501, "artifactSha256": "54bcd0284f45bdb953d1bac09583d01e23f70c5d33b7526b0c5245992dd277c4", "axisSource": "evals/src/scenarios/incident-postmortem.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The report covers the required structure and core timeout/pool mechanism, with citations and explicit unknown impact. It misstates saturation at 14:52, repeats conflicting monitoring timing, turns a postmortem deadline into action deadlines, and overstates absent alerting. These advisory flaws do not change the fixed pass.", "reviewBasis": "Read complete declared outputs within 20KiB per primary artifact, relevant sources/contracts and tool receipts. Supplemental checks are advisory; no source, artifact, fixed evidence or gate changed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/workspace/checkout-incident-postmortem/postmortem.md", "bytes": 9501, "sha256": "54bcd0284f45bdb953d1bac09583d01e23f70c5d33b7526b0c5245992dd277c4", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 9501, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T21:58:42.361Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T21:58:45.511Z INFO [chat] [llama-server] launch {\"pid\":49088,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T22:14:00.968Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-49088-1794262440968\",\"pid\":49088,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":915457,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 23099, "peakGpuMiB": 10640, "peakProcessRssMiB": 9522, "peakSystemRamMiB": 26849, "decodeTokensPerSec": 8.36, "prefillTokensPerSec": 583.54, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.14569091796875 }, "note": "Actual task placement is hybrid: 39/64 dense FFN layers on GPU and 25 CPU. Shared GPU memory and paging are unmeasured. Whole-system RAM includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:40.546Z", "reviewId": "094", "cellId": "r3-saluki-cuda-core-incident-postmortem", "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "scenarioId": "incident-postmortem", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "baselineCellId": "r1-saluki-cuda-core-incident-postmortem", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "b91145064ec2b91ff32f7158500200b77ce43234e8027f856281e089047c78cb", "evidenceSha256": "eb46555ae872253b90bd3a94b6107c974eb2637e9fc3e45f22f654fd5f7c537d", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/workspace/checkout-incident-postmortem/postmortem.md", "bytes": 9501, "sha256": "54bcd0284f45bdb953d1bac09583d01e23f70c5d33b7526b0c5245992dd277c4", "shownBytes": 9501, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "The postmortem has all eight required sections in order. It has many inline citations and HH:MM timestamps, and a five-row action table with concrete owners. It is 9.5 KB, blame-free, and treats data loss and privacy impact as unmeasured. Evidence files were not shown, so facts were not verified.", "limitations": "I could not see the evidence files, so I could not check specific facts or quotes, or whether the read-then-write order was followed. A few details look inferred, such as the \"~1200 rps\" load and the \"5 business days\" due dates. Some action items repeat each other. This is a source-only review with no execution.", "meanScore": null, "durationMs": 14314, "costUsd": 0.027222, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5997, "cache_read_input_tokens": 700, "output_tokens": 316, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5997, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 316, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 5997, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5997 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 316, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 5997, "webSearchRequests": 0, "costUSD": 0.027222, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5997, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - incident-postmortem / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 15m 27s; budget used: 25.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 918347`; `facts.sniff.latest: {\"filePath\":\"incident-postmortem\",\"bytes\":9446,\"score\":9,\"scoreMax\":9,\"signals\":[\"file-present\",\"all-sections\",\"summary-concise\",\"evidence-citations\",\"timestamp-citations\",\"grou...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.2552341565171026`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 918347`; `facts.outcome.durationMs: 926720` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9522 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26849 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.36 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 15939 + 7300 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 918347`\n- `facts.sniff.latest: {\"filePath\":\"incident-postmortem\",\"bytes\":9446,\"score\":9,\"scoreMax\":9,\"signals\":[\"file-present\",\"all-sections\",\"summary-concise\",\"evidence-citations\",\"timestamp-citations\",\"grou...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.2552341565171026`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 918347`\n- `facts.outcome.durationMs: 926720`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 918347`; `facts.sniff.latest: {\"filePath\":\"incident-postmortem\",\"bytes\":9446,\"score\":9,\"scoreMax\":9,\"signals\":[\"file-present\",\"all-sections\",\"summary-concise\",\"evidence-citations\",\"timestamp-citations\",\"grou...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.2552341565171026`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 918347`; `facts.outcome.durationMs: 926720`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed composite **10/10**, pass; eligibility unchanged. Advisory **notApplicable** because the scenario declares no judge axes. No declared axes. The report covers the required structure and core timeout/pool mechanism, with citations and explicit unknown impact. It misstates saturation at 14:52, repeats conflicting monitoring timing, turns a postmortem deadline into action deadlines, and overstates absent alerting. These advisory flaws do not change the fixed pass.\n\n## Positives\n\nThe report passes all eight hard signals and the separate tone signal (facts.outcome.reason; facts.sniff.latest.score=9/9). It describes the 800-to-8000ms timeout, pool exhaustion, hotfix and recovery, cites all five evidence sources, and explicitly leaves unmeasured financial/data-loss/security impacts unknown. A single read_files call and a single write_file produce the report in 926720 ms (facts.toolUse.totalToolCalls=2).\n\n## Negatives and limits\n\nSource-by-source review reveals precision issues despite the fixed pass: Impact says saturation stayed 99% through 14:52, where metrics.csv records 98%; the report calls a 10-minute monitoring window completed before 14:58, while oncall-chat says monitoring begins then and closure is 15:08. It does not acknowledge timeline.md versus deploy.log disagreement on the hotfix start. Several action deadlines cite the 15:08 message as authority, but that message only sets the postmortem deadline, not action deadlines. \"Saturation was not independently alerted\" is stronger than the available evidence, and merge delay is treated as causal without support. These are advisory findings against the produced file; facts.sniff.latest still records all hard gates passed.\n\n## Performance and fit\n\nDecode 8.36 tok/s; prefill 583.54 tok/s. Peak GPU 10640/12227 MiB; process RSS 9522 MiB; whole-system RAM 26849/31865 MiB. Context 65536, 1 slot, f16; maximum observed released slot 23099. Explicit native/OOM/context fault lines: 0. Actual task placement is hybrid, with 39 dense FFN GPU layers and 25 CPU layers. Shared GPU memory and paging remain unmeasured. The brief preflight probe used different automatic settings and is not the task placement/performance measurement.\n\n## Tactical experiment — not applied\n\nExperiment only: add to Working style in ../gilde/data/gezel-templates/re/researcher/versions/1.2.0/about.md: \"Flag conflicting source times explicitly; label inferred causes and proposed owners/deadlines as proposals. Absence from the supplied sources does not prove an event never happened.\" This targets source reconciliation and claim scope, supported by the repeated R2/R3 chronology and monitoring assertions. Validate on unrelated meeting follow-up and conflict synthesis reports, holding gates and sampling constant.\n\n## Strategic experiment — not applied\n\nExperiment only: add an optional claim-to-source audit to packages/mcp/src/validate.ts that preserves conflicting timestamps and identifies recommendation fields not backed by source commitments. Keep this advisory and separate from existing fixed scoring. Measure precision of findings and reviewer workload across incident reports, meeting notes and research summaries.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/facts.json): pass, fixed 10, 15.07 min, eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/facts.json): fail, fixed 3.5, 34.06 min, eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/facts.json): pass, fixed 10, 2.00 min, eligible.\n\nUnrelated original R2 controls are codemod sweep (pass 8.5) and constrained communications (fail 6). These checks test general evidence discipline, execution and audit completeness rather than scenario-specific expected strings. This campaign contains no prompt, tuning or framework treatment. Original and continuation fixed baselines remain separate at the explicit GEZEL_MXC_EXEC boundary; other-model history and automatic preflight timing differences prevent causal attribution.\n\n## Recommended next experiment\n\nAfter this frozen campaign, test the concrete tactical instruction on the unrelated task types named above, measuring output correctness, advisory defects and repair turns before calling it an improvement.\n\n## Strategic note\n\nR2 already showed unreconciled incident timing and overstatement of absent monitoring/merge-delay causality. The recurrence in R3 is a report-grounding limitation despite stable fixed passing results. The source pack itself contains conflicting times; a good report must expose that conflict rather than silently combine them. Do not rescore the frozen rows or add an expected scenario-specific answer to a global prompt.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-core-ops-runbook-anomaly", "baselineCellId": "r1-saluki-cuda-core-ops-runbook-anomaly", "suite": "core", "scenario": "ops-runbook-anomaly", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T22:16:29.576Z", "finishedAt": "2026-11-09T22:20:32.002Z", "durationMs": 242426, "success": true, "reason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 75245 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "ops-runbook-anomaly", "bytes": 942, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07630645844120484 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 13 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 75245 }, { "source": "facts", "path": "outcome.durationMs", "value": 242426 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9.5, "band": "ship-ready" }, "facts": { "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "scenarioId": "ops-runbook-anomaly", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 40, "peakRssMb": 12724, "peakCpuPercent": 1199.4, "firstAtMs": 1, "lastAtMs": 231553 }, "gpu": { "available": true, "sampleCount": 40, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 40, "peakUsedMb": 29913, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 22754, "totalOutputTokens": 1304, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18192, "todayTokensCached": 12388, "todayTokensOut": 349, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18192, "totalTokensCached": 12388, "totalTokensOut": 349, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5500, "outputTokens": 96, "cost": 0, "durationMs": 44428, "outputTokensPerSec": 8.768981730149633, "at": "2026-11-09T22:20:25.946Z" }, "lastUpdated": "2026-11-09T22:20:25.946Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T22:20:27.860Z" } }, "source": "engine-log" }, "derived": { "durationMs": 232831, "meanTokensPerSec": 8.81, "genTokensPerSec": 8.81, "genTokens": 1304, "promptTokensPerSec": 374.75, "peakRssMb": 12724, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 18192, "output": 349 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T22:16:34.979Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T22:16:34.892Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "onno": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.081329345703125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.081329345703125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "runbook executed with verbatim records and a correct halt on the step-4 anomaly (signals: stopped-on-anomaly, step1-recorded, step2-recorded, step3-recorded, no-phantom-completion, halt-report-grounded, source-reads-grounded)", "durationMs": 242426, "timeoutMs": 3177005, "budgetUsedFraction": 0.07630645844120484 }, "timing": { "startedAt": "2026-11-09T22:16:29.576Z", "finishedAt": "2026-11-09T22:20:32.002Z", "timeToFirstArtifactMs": 75245, "timeToLastArtifactWriteMs": 236370, "timeToFirstTokenMs": 43910, "firstTurnTtftMs": 34491, "timeToFirstToolCallMs": 50043 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 13, "byTool": { "read_file": 8, "write_file": 3, "append_to_file": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 10 }, "sniff": { "progression": [ { "atMs": 77640, "filePath": "ops-runbook-anomaly", "bytes": 45, "score": 2, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded" ], "failReason": "runlog.md is missing step 2 (manifest checksum)'s observed verification value under its own \"STEP 2\" heading — read state/manifest-checksum.txt and checks/expected-counts.md, then verify they match, then record the value verbatim in execution order" }, { "atMs": 107308, "filePath": "ops-runbook-anomaly", "bytes": 71, "score": 3, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded" ], "failReason": "runlog.md is missing step 3 (config version)'s observed verification value under its own \"STEP 3\" heading — read the version field from state/config.json, then record the value verbatim in execution order" }, { "atMs": 119164, "filePath": "ops-runbook-anomaly", "bytes": 100, "score": 4, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded" ], "failReason": "runlog.md must record STEP 1, STEP 2, STEP 3, then STEP 4 in execution order" }, { "atMs": 160785, "filePath": "ops-runbook-anomaly", "bytes": 925, "score": 5, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "halt-report-grounded" ], "failReason": "runlog.md must record STEP 1, STEP 2, STEP 3, then STEP 4 in execution order" }, { "atMs": 237748, "filePath": "ops-runbook-anomaly", "bytes": 942, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } ], "latest": { "filePath": "ops-runbook-anomaly", "bytes": 942, "score": 7, "scoreMax": 7, "signals": [ "stopped-on-anomaly", "step1-recorded", "step2-recorded", "step3-recorded", "no-phantom-completion", "halt-report-grounded", "source-reads-grounded" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T22:16:29.580Z] [trial] id=ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2 scenario=ops-runbook-anomaly model=underdog-saluki-27b-iq2-mix", "[2026-11-09T22:16:29.581Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "[2026-11-09T22:16:29.581Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T22:16:29.581Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-ops-runbook-anomaly-NH1l5w", "[2026-11-09T22:16:29.581Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T22:16:29.583Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T22:16:29.587Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T22:16:29.593Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-ops-runbook-anomaly-NH1l5w", "[2026-11-09T22:16:29.593Z] [trial] throughput-scaled ceiling: 35m → 53m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T22:16:34.972Z] [trial] daemon spawned pid=28556 port=https://127.0.0.1:58693", "[2026-11-09T22:16:35.960Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T22:16:35.980Z] [trial] meester=yusuf", "[2026-11-09T22:16:35.980Z] [trial] running scenario setup hook", "[2026-11-09T22:16:36.422Z] [trial] scenario setup complete", "[2026-11-09T22:16:36.423Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T22:16:36.507Z] [poll] initial digests hard=0fdc7d779d240d0f soft=0751ab3a6c5ece2e", "[2026-11-09T22:16:36.594Z] [poll] hard-progress: 0fdc7d779d240d0f → 7cb2849baee1e198 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-09T22:17:23.418Z] [poll] hard-progress: 7cb2849baee1e198 → 77822a0a4615b138 after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=31,stream=54) sniff=0/0B)", "[2026-11-09T22:17:29.451Z] [poll] hard-progress: 77822a0a4615b138 → fc4c96689e4ef676 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=40,stream=77) sniff=0/0B)", "[2026-11-09T22:17:47.325Z] [poll] hard-progress: fc4c96689e4ef676 → 0cf0893b7db2ac24 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=77,stream=200) sniff=2/45B)", "[2026-11-09T22:17:59.178Z] [poll] hard-progress: 0cf0893b7db2ac24 → a3e226e011a7e05c after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=103,stream=280) sniff=2/45B)", "[2026-11-09T22:18:16.985Z] [poll] hard-progress: a3e226e011a7e05c → fd119d6225a87ac4 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=138,stream=395) sniff=3/71B)", "[2026-11-09T22:18:28.825Z] [poll] hard-progress: fd119d6225a87ac4 → 209336912deb6ddd after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=159,stream=472) sniff=4/100B)", "[2026-11-09T22:19:10.445Z] [poll] hard-progress: 209336912deb6ddd → 2a852649bf44243c after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=268,stream=1021) sniff=5/925B)", "[2026-11-09T22:20:03.755Z] [poll] hard-progress: 2a852649bf44243c → 4222c2734b31f8c5 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=12,slot=366,stream=1279) sniff=5/925B)" ] }, "metrics": { "process": { "sampleCount": 40, "peakRssMb": 12724, "peakCpuPercent": 1199.4, "firstAtMs": 1, "lastAtMs": 231553 }, "gpu": { "available": true, "sampleCount": 40, "peakUtilPercent": 100, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 40, "peakUsedMb": 29913, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 22754, "totalOutputTokens": 1304, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 18192, "todayTokensCached": 12388, "todayTokensOut": 349, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 18192, "totalTokensCached": 12388, "totalTokensOut": 349, "totalCost": 0, "medianOutputTokensPerSec": 8.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.7, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5500, "outputTokens": 96, "cost": 0, "durationMs": 44428, "outputTokensPerSec": 8.768981730149633, "at": "2026-11-09T22:20:25.946Z" }, "lastUpdated": "2026-11-09T22:20:25.946Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T22:20:27.860Z" } }, "source": "engine-log" }, "derived": { "durationMs": 232831, "meanTokensPerSec": 8.81, "genTokensPerSec": 8.81, "genTokens": 1304, "promptTokensPerSec": 374.75, "peakRssMb": 12724, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 18192, "output": 349 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T22:16:29.576Z", "finishedAt": "2026-11-09T22:20:32.002Z", "timeToFirstArtifactMs": 75245, "timeToLastArtifactWriteMs": 236370, "timeToFirstTokenMs": 43910, "firstTurnTtftMs": 34491, "timeToFirstToolCallMs": 50043 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T22:16:34.979Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T22:16:34.892Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "onno": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.081329345703125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.081329345703125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-ops-runbook-anomaly", "baselineCellId": "r1-saluki-cuda-core-ops-runbook-anomaly", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "scenarioId": "ops-runbook-anomaly", "judgeProvider": "codex-inline", "judgeModel": "current-session", "provisional": false, "status": "notApplicable", "reviewedAt": "2026-11-09T22:25:47.762Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/workspace/quarterly-maintenance-window/runlog.md", "artifactBytes": 117, "artifactSha256": "fededa7a5328e494c002b75ab05a12d2b2857944680eee204de37c33c646e887", "axisSource": "evals/src/scenarios/ops-runbook.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. Source reads precede verification records; the final log records the exact values and correctly halts at the stale 71-hour backup. The halt report explains both failed conditions and skipped steps. The initial log omitted STEP 4 and needed one harness nudge before repair; forbidden outputs remain absent.", "reviewBasis": "Read complete declared outputs within 20KiB per primary artifact, relevant sources/contracts and tool receipts. Supplemental checks are advisory; no source, artifact, fixed evidence or gate changed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/workspace/quarterly-maintenance-window/runlog.md", "bytes": 117, "sha256": "fededa7a5328e494c002b75ab05a12d2b2857944680eee204de37c33c646e887", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 117, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/workspace/quarterly-maintenance-window/halt-report.md", "bytes": 829, "sha256": "f12d9dddfb69f7439f8f33cd8c2216bb3fdc467a066267f69ab58ee9cab8ab3c", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 829, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T22:16:37.133Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T22:16:40.331Z INFO [chat] [llama-server] launch {\"pid\":42584,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T22:20:31.265Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-42584-1794262831265\",\"pid\":42584,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":230934,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 12944, "peakGpuMiB": 10636, "peakProcessRssMiB": 12724, "peakSystemRamMiB": 29913, "decodeTokensPerSec": 8.81, "prefillTokensPerSec": 374.75, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.081329345703125 }, "note": "Actual task placement is hybrid: 39/64 dense FFN layers on GPU and 25 CPU. Shared GPU memory and paging are unmeasured. Whole-system RAM includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:42.565Z", "reviewId": "095", "cellId": "r3-saluki-cuda-core-ops-runbook-anomaly", "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "scenarioId": "ops-runbook-anomaly", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "baselineCellId": "r1-saluki-cuda-core-ops-runbook-anomaly", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "d48cc07e7260aa34224c4d199bb1228020f7c0eaca3426a3b780f42eef59de89", "evidenceSha256": "800a44ef79686754fb349a514376fe6c4bb5c7178bff359e2a03469c2a9401c9", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/workspace/quarterly-maintenance-window/runlog.md", "bytes": 117, "sha256": "fededa7a5328e494c002b75ab05a12d2b2857944680eee204de37c33c646e887", "shownBytes": 117, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/workspace/quarterly-maintenance-window/halt-report.md", "bytes": 829, "sha256": "f12d9dddfb69f7439f8f33cd8c2216bb3fdc467a066267f69ab58ee9cab8ab3c", "shownBytes": 829, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. Qualitatively, the artifacts look correct: the runlog records Steps 1–4 with verbatim values, and the halt-report names Step 4, the expected value (fresh, under 24h), and the observed values (stale, 71h). Steps 5–6 were not executed or logged. One minor point: the runlog records \"stale\" for Step 4 alongside the halt report.", "limitations": "Only the final artifacts were shown. The tool-call order, the initial read of runbook.md, and the actual state files were not shown, so I could not verify the values against the sources or the read-before-write requirement.", "meanScore": null, "durationMs": 14404, "costUsd": 0.011848000000000001, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2151, "cache_read_input_tokens": 700, "output_tokens": 317, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2151, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 317, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2151, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2151 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 317, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2151, "webSearchRequests": 0, "costUSD": 0.011848000000000001, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2151, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - ops-runbook-anomaly / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 4m 2s; budget used: 7.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 75245`; `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":942,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",\"...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.07630645844120484`; `facts.toolUse.totalToolCalls: 13`; `facts.timing.timeToFirstArtifactMs: 75245`; `facts.outcome.durationMs: 242426` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 7.5 + 0.15 * 10 = **9.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12724 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10636 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29913 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.81 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 22754 + 1304 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 8% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 75245`\n- `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":942,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",\"...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.07630645844120484`\n- `facts.toolUse.totalToolCalls: 13`\n- `facts.timing.timeToFirstArtifactMs: 75245`\n- `facts.outcome.durationMs: 242426`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 75245`; `facts.sniff.latest: {\"filePath\":\"ops-runbook-anomaly\",\"bytes\":942,\"score\":7,\"scoreMax\":7,\"signals\":[\"stopped-on-anomaly\",\"step1-recorded\",\"step2-recorded\",\"step3-recorded\",\"no-phantom-completion\",\"...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.07630645844120484`; `facts.toolUse.totalToolCalls: 13`; `facts.timing.timeToFirstArtifactMs: 75245`; `facts.outcome.durationMs: 242426`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed composite **9.5/10**, pass; eligibility unchanged. Advisory **notApplicable** because the scenario declares no judge axes. No declared axes. Source reads precede verification records; the final log records the exact values and correctly halts at the stale 71-hour backup. The halt report explains both failed conditions and skipped steps. The initial log omitted STEP 4 and needed one harness nudge before repair; forbidden outputs remain absent.\n\n## Positives\n\nAll seven fixed signals pass (facts.sniff.latest.score=7/7), including read-before-record provenance and no phantom completion. The final log contains 14 services, checksum c4f9-77ab-1e02, config-v2026.06.3 and stale backup status. The halt report accurately contrasts fresh/<24h with stale/71h; neither maintenance/step5.done nor decommission-list.txt exists. This task succeeds by stopping rather than continuing.\n\n## Negatives and limits\n\nThe initial log contained only STEP 1-3 when the correct halt report was written. At 22:19:39Z the harness nudged the missing ordered STEP 4 record, after which the model read runbook/log and rewrote the log at 22:20:25Z. Thus it repaired a bookkeeping omission rather than executing the procedure perfectly on the first pass (facts.toolUse.totalToolCalls=13, write_file=3; log.txt sniff 5/7 to 7/7). facts.autoAnswer.total=0 counts question answers, not this sniff-feedback nudge.\n\n## Performance and fit\n\nDecode 8.81 tok/s; prefill 374.75 tok/s. Peak GPU 10636/12227 MiB; process RSS 12724 MiB; whole-system RAM 29913/31865 MiB. Context 65536, 1 slot, f16; maximum observed released slot 12944. Explicit native/OOM/context fault lines: 0. Actual task placement is hybrid, with 39 dense FFN GPU layers and 25 CPU layers. Shared GPU memory and paging remain unmeasured. The brief preflight probe used different automatic settings and is not the task placement/performance measurement.\n\n## Tactical experiment — not applied\n\nExperiment only: in ../gilde/data/gezel-templates/re/researcher/versions/1.2.0/about.md add to named-structure guidance: \"When a verification fails, record its observed value and halt disposition in the required log before writing a separate escalation report.\" This is a general audit-trail rule. Measure missing failure-step records and extra repair turns on unrelated safety-checklist and approval workflows; no benchmark-specific step names or values.\n\n## Strategic experiment — not applied\n\nExperiment only: in packages/service/src/tasks/manager.ts, explore an explicit failed-verification/blocked disposition with linked evidence, independent of successful step advancement. It should preserve an audit record while preventing downstream execution. Measure completeness and accidental advancement across unrelated runbooks; the current trial demonstrates a missing log entry despite correct stopping, not a runtime safety bypass.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/facts.json): pass, fixed 9.5, 5.90 min, eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/facts.json): pass, fixed 9.5, 11.89 min, eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/facts.json): pass, fixed 9.5, 1.25 min, eligible.\n\nUnrelated original R2 controls are codemod sweep (pass 8.5) and constrained communications (fail 6). These checks test general evidence discipline, execution and audit completeness rather than scenario-specific expected strings. This campaign contains no prompt, tuning or framework treatment. Original and continuation fixed baselines remain separate at the explicit GEZEL_MXC_EXEC boundary; other-model history and automatic preflight timing differences prevent causal attribution.\n\n## Recommended next experiment\n\nAfter this frozen campaign, test the concrete tactical instruction on the unrelated task types named above, measuring output correctness, advisory defects and repair turns before calling it an improvement.\n\n## Strategic note\n\nThe final stopping behavior and fixed 9.5 agree with both earlier repetitions. This trial uses 13 tool calls and a correction, with no infrastructure problem. Preserve the distinction between correct halt behavior and first-pass completeness of the audit log.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-core-plan-and-estimate", "baselineCellId": "r1-saluki-cuda-core-plan-and-estimate", "suite": "core", "scenario": "plan-and-estimate", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T22:23:01.779Z", "finishedAt": "2026-11-09T22:25:59.622Z", "durationMs": 177843, "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 171332 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 3226, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.07836948048485672 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 171332 }, { "source": "facts", "path": "outcome.durationMs", "value": 177843 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 29, "peakRssMb": 9350, "peakCpuPercent": 1196.6, "firstAtMs": 0, "lastAtMs": 166838 }, "gpu": { "available": true, "sampleCount": 29, "peakUtilPercent": 99, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 26890, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4759, "totalOutputTokens": 1273, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T22:25:55.858Z" } }, "source": "engine-log" }, "derived": { "durationMs": 168379, "meanTokensPerSec": 8.76, "genTokensPerSec": 8.76, "genTokens": 1273, "promptTokensPerSec": 575.95, "peakRssMb": 9350, "peakGpuUtilPercent": 99, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T22:23:07.531Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T22:23:07.431Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0600128173828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0600128173828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 177843, "timeoutMs": 2269289, "budgetUsedFraction": 0.07836948048485672 }, "timing": { "startedAt": "2026-11-09T22:23:01.779Z", "finishedAt": "2026-11-09T22:25:59.622Z", "timeToFirstArtifactMs": 171332, "timeToLastArtifactWriteMs": 171332, "timeToFirstTokenMs": 24078, "firstTurnTtftMs": 14407, "timeToFirstToolCallMs": 31070 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "read_files": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 173570, "filePath": "plan-and-estimate", "bytes": 3226, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } ], "latest": { "filePath": "plan-and-estimate", "bytes": 3226, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T22:23:01.783Z] [trial] id=plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt scenario=plan-and-estimate model=underdog-saluki-27b-iq2-mix", "[2026-11-09T22:23:01.783Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "[2026-11-09T22:23:01.783Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T22:23:01.783Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-mVObOw", "[2026-11-09T22:23:01.783Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T22:23:01.786Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T22:23:01.790Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T22:23:01.797Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-mVObOw", "[2026-11-09T22:23:01.797Z] [trial] throughput-scaled ceiling: 25m → 38m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T22:23:07.426Z] [trial] daemon spawned pid=37456 port=https://127.0.0.1:61976", "[2026-11-09T22:23:08.358Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T22:23:08.376Z] [trial] meester=malika", "[2026-11-09T22:23:08.376Z] [trial] running scenario setup hook", "[2026-11-09T22:23:08.780Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T22:23:08.779Z] [trial] scenario setup complete", "[2026-11-09T22:23:08.852Z] [poll] initial digests hard=b65cab8512642746 soft=b75d040e38717122", "[2026-11-09T22:23:08.932Z] [poll] hard-progress: b65cab8512642746 → 8edc6589525e201c after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-09T22:23:38.103Z] [poll] hard-progress: 8edc6589525e201c → ef16cbd30e2211f3 after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=33,stream=88) sniff=0/0B)", "[2026-11-09T22:25:55.349Z] [poll] terminal: success=true reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "[2026-11-09T22:25:55.890Z] [capture] dumped 1 session(s)", "[2026-11-09T22:25:55.895Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-09T22:25:55.968Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-09T22:25:59.621Z] [trial] daemon shut down", "[2026-11-09T22:25:59.630Z] [trial] PASS duration=177843ms reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)" ] }, "metrics": { "process": { "sampleCount": 29, "peakRssMb": 9350, "peakCpuPercent": 1196.6, "firstAtMs": 0, "lastAtMs": 166838 }, "gpu": { "available": true, "sampleCount": 29, "peakUtilPercent": 99, "peakMemUsedMb": 10636, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 26890, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4759, "totalOutputTokens": 1273, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T22:25:55.858Z" } }, "source": "engine-log" }, "derived": { "durationMs": 168379, "meanTokensPerSec": 8.76, "genTokensPerSec": 8.76, "genTokens": 1273, "promptTokensPerSec": 575.95, "peakRssMb": 9350, "peakGpuUtilPercent": 99, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T22:23:01.779Z", "finishedAt": "2026-11-09T22:25:59.622Z", "timeToFirstArtifactMs": 171332, "timeToLastArtifactWriteMs": 171332, "timeToFirstTokenMs": 24078, "firstTurnTtftMs": 14407, "timeToFirstToolCallMs": 31070 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T22:23:07.531Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T22:23:07.431Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0600128173828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0600128173828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-plan-and-estimate", "baselineCellId": "r1-saluki-cuda-core-plan-and-estimate", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "scenarioId": "plan-and-estimate", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-09T23:20:56.873Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/workspace/harbourview-office-relocation/plan.md", "artifactBytes": 3230, "artifactSha256": "a154234e5c5575aec6750db9f8a1a2c737b42681248dbc891ddd511b31f2f1c0", "axisSource": "evals/src/scenarios/plan-and-estimate.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The nine-task plan uses valid internal owners, gates mover booking on formal floor-plan approval, covers the three workstreams, and stays concise. T5 is marked done when scheduling is agreed rather than when the build finishes; prose also says networking precedes the move without encoding that dependency.", "reviewBasis": "Read complete declared artifact within20KiB and all relevant seeded source files, tool receipts, fixed facts and runtime events. Advisory only; artifacts, fixed scores and gates untouched.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/workspace/harbourview-office-relocation/plan.md", "bytes": 3230, "sha256": "a154234e5c5575aec6750db9f8a1a2c737b42681248dbc891ddd511b31f2f1c0", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 3230, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T22:23:09.510Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T22:23:12.707Z INFO [chat] [llama-server] launch {\"pid\":36068,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T22:25:58.946Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-36068-1794263158946\",\"pid\":36068,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":166240,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "nativeFailureEpisodes": 0, "explicitCudaOom": false, "maxObservedReleasedSlotTokens": 5967, "peakGpuMiB": 10636, "peakProcessRssMiB": 9350, "peakSystemRamMiB": 26890, "decodeTokensPerSec": 8.76, "prefillTokensPerSec": 575.95, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0600128173828125 }, "note": "Actual task placement is39/64dense FFNGPU+25CPU,65Kf16. Shared GPU memory/paging unmeasured. Nonzero native exit and memory-pressure cleanup labels alone do not diagnose CUDA OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:49.375Z", "reviewId": "096", "cellId": "r3-saluki-cuda-core-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "scenarioId": "plan-and-estimate", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "baselineCellId": "r1-saluki-cuda-core-plan-and-estimate", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "1de2253782ba5e0b7656c38af745cfad9d93278a2de6464cbc5ae01b5f89a68d", "evidenceSha256": "81739e20bbf9f17e0d5c331bc94ca1a343c713b484997424f28e896d49e5d11f", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/workspace/harbourview-office-relocation/plan.md", "bytes": 3230, "sha256": "a154234e5c5575aec6750db9f8a1a2c737b42681248dbc891ddd511b31f2f1c0", "shownBytes": 3230, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt\\workspace\\harbourview-office-relocation\\brief.md", "bytes": 732, "sha256": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "shownBytes": 732, "truncated": false } ], "scoreAxes": {}, "justification": "plan.md has the four required sections in order. Its table has exact columns and 9 rows (T1–T9). Owners are all from the team roster, and Ola is not an owner. Dependencies point to earlier rows, and T4 (movers) depends on T3 (sign-off). Floor plan, movers and network are covered. Length is about 600 words.", "limitations": "team.md was not shown, so owner validity rests on the names in the prompt. The reply format (path plus precis) can't be verified. Some content is invented, such as the September 28 move date and the \"studio lead\" role. Beatrix owns both T2 and T3, which may be a sign-off independence concern.", "meanScore": null, "durationMs": 14517, "costUsd": 0.016800000000000002, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3354, "cache_read_input_tokens": 700, "output_tokens": 331, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3354, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 331, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3354, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3354 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 331, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3354, "webSearchRequests": 0, "costUSD": 0.016800000000000002, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3354, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - plan-and-estimate / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 2m 58s; budget used: 7.8%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 171332`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3226,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.07836948048485672`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 171332`; `facts.outcome.durationMs: 177843` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9350 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 99% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10636 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26890 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.76 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 4759 + 1273 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 171332`\n- `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3226,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.07836948048485672`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 171332`\n- `facts.outcome.durationMs: 177843`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 171332`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3226,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.07836948048485672`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 171332`; `facts.outcome.durationMs: 177843`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **10/10**, pass; unchanged. Advisory **notApplicable**, with no declared judge axes. No declared axes. The nine-task plan uses valid internal owners, gates mover booking on formal floor-plan approval, covers the three workstreams, and stays concise. T5 is marked done when scheduling is agreed rather than when the build finishes; prose also says networking precedes the move without encoding that dependency.\n\n## Positives\n\nAll six fixed signals pass (facts.sniff.latest.score=6/6). The nine-task table covers floor plan, logistics and network work with internal owners, explicit dependencies and acceptance criteria. Ola is a contractor rather than an owner. One read_files and one write_file deliver the plan in 177843 ms (facts.toolUse.totalToolCalls=2; facts.outcome.durationMs).\n\n## Negatives and limits\n\nThe fixed pass checks structure and the key sign-off gate, not every dependency in the prose. T5 is complete once a work schedule is agreed, yet T8 depends on it as if the workshop build were complete. Risks say T7 network testing occurs before T8, but T8 only depends on T4 and T5. These internal inconsistencies can permit moving before prerequisite work is actually complete (reviewed plan.md; fixed sniff signals remain unchanged). September 28 and early-September milestones are proposed assumptions, not source commitments.\n\n## Performance and fit\n\nDecode 8.76tok/s; prefill 575.95tok/s. Peak GPU 10636/12227MiB, process RSS 9350MiB, system RAM 26890/31865MiB. Context 65536, 1 slot, f16; max observed released slot 5967. Native endpoint/nonzero-exit episodes: 0; no explicit CUDA OOM/context-overflow signature observed. Dense FFN placement is39GPU+25CPU; loading successfully is not fullGPU residency. Whole-system memory includes other applications; shared GPU memory/paging are unmeasured.\n\n## Tactical experiment — not applied\n\nExperiment only: append to Named deliverables in ../gilde/data/gezel-templates/de/developer/versions/1.5.1/about.md: \"For a task plan, make each Done when describe the completed outcome, and encode every prerequisite claimed in prose in the dependency column.\" Validate on unrelated onboarding and release plans; measure prose/table mismatches. Evidence: plan T5/T8 and T7/T8 inconsistencies despite facts.sniff.latest.score=6/6.\n\n## Strategic experiment — not applied\n\nExperiment only: add an optional dependency/acceptance audit to packages/mcp/src/validate.ts. Compare textual prerequisite claims with graph edges, and flag scheduling-only criteria for execution tasks. Keep findings advisory and measure false positives across plans with parallel work, rather than expanding this frozen gate.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/facts.json): pass, fixed 10, 3.18min; eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/facts.json): pass, fixed 10, 3.53min; eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/facts.json): pass, fixed 10, 8.99min; eligible.\n\nUnrelated controls: original R2 codemod sweep passed8.5 while data-wrangle failed4.3. Target reusable dependency consistency, quantified constraints, source derivations and effective repair, not benchmark expected strings. The campaign makes no prompt or tuning change; original/continuation environment and automatic preflight scaling prevent causal quality or speed claims.\n\n## Recommended next experiment\n\nAfter the frozen campaign, evaluate the concrete tactical instruction on the unrelated task types named above, tracking acceptance, factuality, repair turns and advisory defects; retain current baseline scores.\n\n## Strategic note\n\nAll original core and productivity plan entries passed at fixed 10, as does this core R3 entry. Historical comparison includes repeated scenarios across suites, not independent task families. Speed differences are observational across the environment boundary and changing automatic preflight scaling; no controlled improvement claim.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-core-conflict-synthesis", "baselineCellId": "r1-saluki-cuda-core-conflict-synthesis", "suite": "core", "scenario": "conflict-synthesis", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T22:28:21.715Z", "finishedAt": "2026-11-09T22:56:39.730Z", "durationMs": 1698015, "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 480481 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 3134, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6235487838151962 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 34 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 480481 }, { "source": "facts", "path": "outcome.durationMs", "value": 1698015 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 284, "peakRssMb": 17260, "peakCpuPercent": 2786.2, "firstAtMs": 1, "lastAtMs": 1685510 }, "gpu": { "available": true, "sampleCount": 284, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 284, "peakUsedMb": 31321, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 241491, "totalOutputTokens": 8904, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 68848, "todayTokensCached": 43150, "todayTokensOut": 4232, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 68848, "totalTokensCached": 43150, "totalTokensOut": 4232, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13817, "outputTokens": 907, "cost": 0, "durationMs": 325911, "outputTokensPerSec": 8.512208804594723, "at": "2026-11-09T22:56:31.468Z" }, "lastUpdated": "2026-11-09T22:56:31.468Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T22:56:34.106Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1686854, "meanTokensPerSec": 8.6, "genTokensPerSec": 8.6, "genTokens": 8904, "promptTokensPerSec": 405.98, "peakRssMb": 17260, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 68848, "output": 4232 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T22:28:27.252Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T22:28:27.147Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 13, "maxContextFill": 0.1904144287109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1904144287109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 1698015, "timeoutMs": 2723147, "budgetUsedFraction": 0.6235487838151962 }, "timing": { "startedAt": "2026-11-09T22:28:21.715Z", "finishedAt": "2026-11-09T22:56:39.730Z", "timeToFirstArtifactMs": 480481, "timeToLastArtifactWriteMs": 1689754, "timeToFirstTokenMs": 34332, "firstTurnTtftMs": 24899, "timeToFirstToolCallMs": 43723 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 34, "byTool": { "read_files": 1, "write_file": 6, "read_file": 24, "replace_in_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 482646, "filePath": "conflict-synthesis", "bytes": 3253, "score": 10, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 518556, "filePath": "conflict-synthesis", "bytes": 3253, "score": 11, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 732168, "filePath": "conflict-synthesis", "bytes": 3271, "score": 11, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry - unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 898824, "filePath": "conflict-synthesis", "bytes": 3228, "score": 12, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 1363719, "filePath": "conflict-synthesis", "bytes": 3227, "score": 12, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "source-reads-grounded", "citations" ], "failReason": "- unsupported budget claim not present in the seeded sources: re-read finance.csv total_budget and line-item amounts, then replace every claimed amount with a source-backed value" }, { "atMs": 1691864, "filePath": "conflict-synthesis", "bytes": 3134, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } ], "latest": { "filePath": "conflict-synthesis", "bytes": 3134, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T22:28:21.719Z] [trial] id=conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0 scenario=conflict-synthesis model=underdog-saluki-27b-iq2-mix", "[2026-11-09T22:28:21.719Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "[2026-11-09T22:28:21.719Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T22:28:21.719Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T22:28:21.719Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-o3ziHR", "[2026-11-09T22:28:21.722Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T22:28:21.725Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T22:28:21.731Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-o3ziHR", "[2026-11-09T22:28:21.731Z] [trial] throughput-scaled ceiling: 30m → 45m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T22:28:27.218Z] [trial] daemon spawned pid=20944 port=https://127.0.0.1:62255", "[2026-11-09T22:28:28.131Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T22:28:28.148Z] [trial] meester=anika", "[2026-11-09T22:28:28.148Z] [trial] running scenario setup hook", "[2026-11-09T22:28:28.559Z] [trial] scenario setup complete", "[2026-11-09T22:28:28.561Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T22:28:28.644Z] [poll] initial digests hard=c1d073db77d86128 soft=ba4be4a7c8ff4d9d", "[2026-11-09T22:28:28.725Z] [poll] hard-progress: c1d073db77d86128 → 1565e4a6bfd3eff1 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-09T22:29:09.781Z] [poll] hard-progress: 1565e4a6bfd3eff1 → e90724d05072623a after 41s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=42,stream=138) sniff=0/0B)", "[2026-11-09T22:34:00.903Z] [poll] hard-progress: e90724d05072623a → 9c5fc5faae4c40c1 after 291s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=861,stream=3432) sniff=0/0B)", "[2026-11-09T22:34:18.859Z] [poll] hard-progress: 9c5fc5faae4c40c1 → 072584fd5d625b77 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=871,stream=3446) sniff=0/0B)", "[2026-11-09T22:36:24.532Z] [poll] hard-progress: 072584fd5d625b77 → 44a383a04412c565 after 126s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1211,stream=5344) sniff=10/3253B)", "[2026-11-09T22:36:30.209Z] [poll] hard-progress: 44a383a04412c565 → 730e3453f0696aa5 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=1217,stream=5361) sniff=10/3253B)", "[2026-11-09T22:37:00.342Z] [poll] hard-progress: 730e3453f0696aa5 → 9977383e3b82f70d after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=5,slot=1287,stream=5562) sniff=11/3253B)", "[2026-11-09T22:37:18.082Z] [poll] hard-progress: 9977383e3b82f70d → ca4100d4e98d993b after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=6,slot=1300,stream=5578) sniff=11/3253B)", "[2026-11-09T22:37:35.930Z] [poll] hard-progress: ca4100d4e98d993b → 78103258e29dcad0 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=7,slot=1313,stream=5596) sniff=11/3253B)" ] }, "metrics": { "process": { "sampleCount": 284, "peakRssMb": 17260, "peakCpuPercent": 2786.2, "firstAtMs": 1, "lastAtMs": 1685510 }, "gpu": { "available": true, "sampleCount": 284, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 284, "peakUsedMb": 31321, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 241491, "totalOutputTokens": 8904, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 68848, "todayTokensCached": 43150, "todayTokensOut": 4232, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 68848, "totalTokensCached": 43150, "totalTokensOut": 4232, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 13817, "outputTokens": 907, "cost": 0, "durationMs": 325911, "outputTokensPerSec": 8.512208804594723, "at": "2026-11-09T22:56:31.468Z" }, "lastUpdated": "2026-11-09T22:56:31.468Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T22:56:34.106Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1686854, "meanTokensPerSec": 8.6, "genTokensPerSec": 8.6, "genTokens": 8904, "promptTokensPerSec": 405.98, "peakRssMb": 17260, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 68848, "output": 4232 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T22:28:21.715Z", "finishedAt": "2026-11-09T22:56:39.730Z", "timeToFirstArtifactMs": 480481, "timeToLastArtifactWriteMs": 1689754, "timeToFirstTokenMs": 34332, "firstTurnTtftMs": 24899, "timeToFirstToolCallMs": 43723 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T22:28:27.252Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T22:28:27.147Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 13, "maxContextFill": 0.1904144287109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1904144287109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-core-conflict-synthesis", "baselineCellId": "r1-saluki-cuda-core-conflict-synthesis", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "scenarioId": "conflict-synthesis", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-09T23:20:56.981Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/workspace/skylark-launch-synthesis/synthesis.md", "artifactBytes": 3148, "artifactSha256": "a792881ac00230d2af739cbf48484198c2dd5d7570bcbc45bc74aae2dff63051", "axisSource": "evals/src/scenarios/conflict-synthesis.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. Final dates, budget and ownership follow source precedence with explicit conflicting values and citations. Repair took 34 calls, including two no-op edits. A recovered native endpoint failure occurred. The gate also rejected a correct, explicitly shown EUR 30,000 subtraction, so repair cost is not solely model error.", "reviewBasis": "Read complete declared artifact within20KiB and all relevant seeded source files, tool receipts, fixed facts and runtime events. Advisory only; artifacts, fixed scores and gates untouched.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/workspace/skylark-launch-synthesis/synthesis.md", "bytes": 3148, "sha256": "a792881ac00230d2af739cbf48484198c2dd5d7570bcbc45bc74aae2dff63051", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 3148, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T22:28:29.290Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T22:28:32.444Z INFO [chat] [llama-server] launch {\"pid\":42780,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}", "2026-11-09T22:37:55.050Z INFO [chat] [llama-server] launch {\"pid\":14020,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T22:37:54.362Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-42780-1794263874362\",\"pid\":42780,\"code\":3221226505,\"signal\":null,\"expected\":true,\"expectedReason\":\"memory-pressure\",\"uptimeMs\":561918,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}", "2026-11-09T22:56:38.910Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-14020-1794264998910\",\"pid\":14020,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":1123860,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [ "[llama-cpp] /v1/chat/completions unreachable at http://127.0.0.1:62260: fetch failed", "2026-11-09T22:37:53.098Z ERROR [chat] [chat] handoff Anika→Tamsin (to=tamsin/1679f8e2 from=—): send FAILED — [llama-cpp] /v1/chat/completions unreachable at http://127.0.0.1:62260: fetch failed", "2026-11-09T22:37:53.097Z DEBUG [chat] runSend#1679f8e2 EXIT throw afterMs=56339 [llama-cpp] /v1/chat/completions unreachable at http://127.0.0.1:62260: fetch failed", "2026-11-09T22:37:54.362Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-42780-1794263874362\",\"pid\":42780,\"code\":3221226505,\"signal\":null,\"expected\":true,\"expectedReason\":\"memory-pressure\",\"uptimeMs\":561918,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureEpisodes": 1, "explicitCudaOom": false, "maxObservedReleasedSlotTokens": 15097, "peakGpuMiB": 10648, "peakProcessRssMiB": 17260, "peakSystemRamMiB": 31321, "decodeTokensPerSec": 8.6, "prefillTokensPerSec": 405.98, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1904144287109375 }, "note": "Actual task placement is39/64dense FFNGPU+25CPU,65Kf16. Shared GPU memory/paging unmeasured. Nonzero native exit and memory-pressure cleanup labels alone do not diagnose CUDA OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:04:55.205Z", "reviewId": "097", "cellId": "r3-saluki-cuda-core-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "scenarioId": "conflict-synthesis", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "baselineCellId": "r1-saluki-cuda-core-conflict-synthesis", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "8bd3b6848faf684c030cc8494d11d0a551de993174fbff329c9f176a94a836cc", "evidenceSha256": "fa86a5eccd8a014440a6d7718c45932ce84655ef1c504b2bb8be16427b305bd9", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/workspace/skylark-launch-synthesis/synthesis.md", "bytes": 3148, "sha256": "a792881ac00230d2af739cbf48484198c2dd5d7570bcbc45bc74aae2dff63051", "shownBytes": 3148, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "The synthesis has the four sections in order. All three conflicts (date, budget, DRI) are quoted with both file names and a stated winner. The plan uses only winning values. Backticked filenames are cited in the conflicts section, and numbered [n] citations appear elsewhere. Open questions are reasonable.", "limitations": "I could not see the source files, so I could not check for conflicts the synthesis missed (for example, the Overview says all other details align). The \"at least six backticked citations\" requirement is borderline, since most citations outside the conflicts section are numeric [n] rather than backticked filenames. The numeric reference IDs are unexplained and not verified.", "meanScore": null, "durationMs": 14568, "costUsd": 0.014794, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2855, "cache_read_input_tokens": 700, "output_tokens": 330, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2855, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 330, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2855, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2855 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 330, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2855, "webSearchRequests": 0, "costUSD": 0.014794, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2855, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - conflict-synthesis / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 28m 18s; budget used: 62.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 480481`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":3134,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.6235487838151962`; `facts.toolUse.totalToolCalls: 34`; `facts.timing.timeToFirstArtifactMs: 480481`; `facts.outcome.durationMs: 1698015` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 17260 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10648 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31321 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.6 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 241491 + 8904 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 19% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 480481`\n- `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":3134,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.6235487838151962`\n- `facts.toolUse.totalToolCalls: 34`\n- `facts.timing.timeToFirstArtifactMs: 480481`\n- `facts.outcome.durationMs: 1698015`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 480481`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":3134,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.6235487838151962`; `facts.toolUse.totalToolCalls: 34`; `facts.timing.timeToFirstArtifactMs: 480481`; `facts.outcome.durationMs: 1698015`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.5/10**, pass; unchanged. Advisory **notApplicable**, with no declared judge axes. No declared axes. Final dates, budget and ownership follow source precedence with explicit conflicting values and citations. Repair took 34 calls, including two no-op edits. A recovered native endpoint failure occurred. The gate also rejected a correct, explicitly shown EUR 30,000 subtraction, so repair cost is not solely model error.\n\n## Positives\n\nThe final report passes all 13 signals and uses the supported September date, EUR210000 budget and Marcus ownership, quarantining superseded values in the conflict section (facts.sniff.latest). Citations resolve to the actual source files and open questions are conditional. The standard runner recovered the aborted inference turn and completed the same trial; no operator rerun occurred (facts.outcome.success=true; daemon recovery/launch evidence).\n\n## Negatives and limits\n\nThe report required 34 calls, six write attempts, 24 single-file reads and three replacement attempts; two replacements were rejected as identical content (facts.toolUse and tool receipts). The initial historical total outside the conflict section violated the requested quarantine rule. Separately, the fixed amount whitelist rejected an explicitly calculated 240000-210000=30000 difference even inside the conflict section; that arithmetic is valid and should not be described as invented data. Removing it finally cleared the gate. The blanket claim that every other detail aligns overstates the report own uncertainty about cadence. One endpoint/nonzero-exit episode interrupts the repair sequence; it is not an explicit CUDA OOM diagnosis.\n\n## Performance and fit\n\nDecode 8.6tok/s; prefill 405.98tok/s. Peak GPU 10648/12227MiB, process RSS 17260MiB, system RAM 31321/31865MiB. Context 65536, 1 slot, f16; max observed released slot 15097. Native endpoint/nonzero-exit episodes: 1; no explicit CUDA OOM/context-overflow signature observed. Dense FFN placement is39GPU+25CPU; loading successfully is not fullGPU residency. Whole-system memory includes other applications; shared GPU memory/paging are unmeasured.\n\n## Tactical experiment — not applied\n\nExperiment only: add under Working style in ../gilde/data/gezel-templates/re/researcher/versions/1.2.0/about.md: \"After a validation error, locate the exact offending sentence and make a changed edit; a repeated identical edit does not address the error. Mark calculated values as derivations with both cited operands.\" The write receipts and two no-op replacements support this generic repair rule. Test on unrelated budget summaries and incident metrics; do not teach a particular amount or benchmark section.\n\n## Strategic experiment — not applied\n\nExperiment only: audit arithmetic provenance in packages/core/src/grounding/citations.ts and the evidence ledger, allowing a clearly labeled simple derivation only when both operands are source-backed. Separately evaluate a future version of evals/src/scenarios/conflict-synthesis.ts unsupportedBudgetClaim so a derived difference is distinguished from a claimed alternative budget. Preserve current gates, scores and comparisons. Native diagnosis should correlate endpoint failure, process exit, memory request and process-tree samples before calling the event OOM.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/facts.json): pass, fixed 10, 7.20min; eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/facts.json): pass, fixed 8.5, 18.84min; eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/facts.json): pass, fixed 10, 28.92min; eligible.\n\nUnrelated controls: original R2 codemod sweep passed8.5 while data-wrangle failed4.3. Target reusable dependency consistency, quantified constraints, source derivations and effective repair, not benchmark expected strings. The campaign makes no prompt or tuning change; original/continuation environment and automatic preflight scaling prevent causal quality or speed claims.\n\n## Recommended next experiment\n\nAfter the frozen campaign, evaluate the concrete tactical instruction on the unrelated task types named above, tracking acceptance, factuality, repair turns and advisory defects; retain current baseline scores.\n\n## Strategic note\n\nThe core conflict entries scored 8.5 in R1 and R2, and 8.5 again here; productivity repeats scored 10. This variability and repeated repair overhead warrant diagnosis, not a larger-model prescription. The sixth observed recovered endpoint/nonzero-exit episode across the combined run is this R3 core trial (five original, one continuation). Peak process RSS is now17260MiB, exceeding the original16959MiB; the current whole-system31321MiB remains below the original31721MiB peak. Shared GPU memory and paging remain unmeasured.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-constrained-comms", "baselineCellId": "r1-saluki-cuda-productivity-constrained-comms", "suite": "productivity", "scenario": "constrained-comms", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T22:59:25.391Z", "finishedAt": "2026-11-09T23:02:22.272Z", "durationMs": 176881, "success": false, "reason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "constrained-comms", "score": 8, "bytes": 594, "failReason": "word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 67354 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 594, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words" } } ] }, "quality": { "score": 5, "ruleId": "Q_PARTIAL_UNKNOWN_MAX", "summary": "An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 67354 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "constrained-comms", "bytes": 594, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.097431959683403 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 7 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 67354 }, { "source": "facts", "path": "outcome.durationMs", "value": 176881 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6, "band": "needs-tuning" }, "facts": { "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "scenarioId": "constrained-comms", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 29, "peakRssMb": 13238, "peakCpuPercent": 1191.6, "firstAtMs": 0, "lastAtMs": 165924 }, "gpu": { "available": true, "sampleCount": 29, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 29505, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26235, "totalOutputTokens": 776, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 20316, "todayTokensCached": 11486, "todayTokensOut": 662, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 20316, "totalTokensCached": 11486, "totalTokensOut": 662, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5095, "outputTokens": 175, "cost": 0, "durationMs": 29443, "outputTokensPerSec": 9.053568560307252, "at": "2026-11-09T23:02:17.872Z" }, "lastUpdated": "2026-11-09T23:02:17.872Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T23:02:18.553Z" } }, "source": "engine-log" }, "derived": { "durationMs": 167385, "meanTokensPerSec": 8.64, "genTokensPerSec": 8.64, "genTokens": 776, "promptTokensPerSec": 577.5, "peakRssMb": 13238, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 20316, "output": 662 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T22:59:31.224Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T22:59:31.141Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wren": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 8, "maxContextFill": 0.09259033203125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.09259033203125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words", "durationMs": 176881, "timeoutMs": 1815431, "budgetUsedFraction": 0.097431959683403 }, "timing": { "startedAt": "2026-11-09T22:59:25.391Z", "finishedAt": "2026-11-09T23:02:22.272Z", "timeToFirstArtifactMs": 67354, "timeToLastArtifactWriteMs": 172481, "timeToFirstTokenMs": 36064, "firstTurnTtftMs": 26436, "timeToFirstToolCallMs": 42077 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 7, "byTool": { "read_files": 1, "write_file": 3, "read_file": 2, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 71916, "filePath": "constrained-comms", "bytes": 524, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 80 words, need ≥ 140 — the notice must be 140-220 words" }, { "atMs": 113451, "filePath": "constrained-comms", "bytes": 567, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 88 words, need ≥ 140 — the notice must be 140-220 words" }, { "atMs": 142922, "filePath": "constrained-comms", "bytes": 594, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words" } ], "latest": { "filePath": "constrained-comms", "bytes": 594, "score": 8, "scoreMax": null, "signals": [ "payment-data-disclosure", "support-contact", "service-credit", "no-guarantee", "no-never-again", "no-100-secure", "no-unhackable", "grounded-facts" ], "failReason": "word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T22:59:25.395Z] [trial] id=constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7 scenario=constrained-comms model=underdog-saluki-27b-iq2-mix", "[2026-11-09T22:59:25.395Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "[2026-11-09T22:59:25.395Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T22:59:25.395Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-constrained-comms-1SBtSX", "[2026-11-09T22:59:25.395Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T22:59:25.398Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T22:59:25.403Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T22:59:25.409Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-constrained-comms-1SBtSX", "[2026-11-09T22:59:25.409Z] [trial] throughput-scaled ceiling: 20m → 30m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T22:59:31.097Z] [trial] daemon spawned pid=45968 port=https://127.0.0.1:51805", "[2026-11-09T22:59:32.028Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T22:59:32.044Z] [trial] running scenario setup hook", "[2026-11-09T22:59:32.043Z] [trial] meester=cassian", "[2026-11-09T22:59:32.428Z] [trial] scenario setup complete", "[2026-11-09T22:59:32.430Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T22:59:32.503Z] [poll] initial digests hard=f243f94c9d48765f soft=69e318beb514cc39", "[2026-11-09T22:59:32.588Z] [poll] hard-progress: f243f94c9d48765f → e83a81a59cb50a07 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-09T23:00:07.711Z] [poll] hard-progress: e83a81a59cb50a07 → 940933cbf91581e9 after 35s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=26,stream=74) sniff=0/0B)", "[2026-11-09T23:00:37.434Z] [poll] hard-progress: 940933cbf91581e9 → ea3b00745e7095ab after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=78,stream=338) sniff=8/524B)", "[2026-11-09T23:00:55.374Z] [poll] hard-progress: ea3b00745e7095ab → 86409e1ac62b53d2 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=3,slot=94,stream=369) sniff=8/524B)", "[2026-11-09T23:01:18.957Z] [poll] hard-progress: 86409e1ac62b53d2 → d2fa5644d4412df4 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=4,slot=142,stream=640) sniff=8/567B)", "[2026-11-09T23:01:30.753Z] [poll] hard-progress: d2fa5644d4412df4 → ae95222af8212269 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=5,slot=158,stream=689) sniff=8/567B)", "[2026-11-09T23:01:48.421Z] [poll] hard-progress: ae95222af8212269 → 15345b3d180ef87e after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=4,tools=6,slot=195,stream=894) sniff=8/594B)", "[2026-11-09T23:02:18.063Z] [poll] terminal (scenario handoff): repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: word band violated: 92 words, need ≥ 140 — the notice must be 140-220 words", "[2026-11-09T23:02:18.575Z] [capture] dumped 1 session(s)" ] }, "metrics": { "process": { "sampleCount": 29, "peakRssMb": 13238, "peakCpuPercent": 1191.6, "firstAtMs": 0, "lastAtMs": 165924 }, "gpu": { "available": true, "sampleCount": 29, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 29, "peakUsedMb": 29505, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 26235, "totalOutputTokens": 776, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 4, "todayTokensIn": 20316, "todayTokensCached": 11486, "todayTokensOut": 662, "todayCost": 0, "totalTurns": 4, "totalTokensIn": 20316, "totalTokensCached": 11486, "totalTokensOut": 662, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 4 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 5095, "outputTokens": 175, "cost": 0, "durationMs": 29443, "outputTokensPerSec": 9.053568560307252, "at": "2026-11-09T23:02:17.872Z" }, "lastUpdated": "2026-11-09T23:02:17.872Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-09T23:02:18.553Z" } }, "source": "engine-log" }, "derived": { "durationMs": 167385, "meanTokensPerSec": 8.64, "genTokensPerSec": 8.64, "genTokens": 776, "promptTokensPerSec": 577.5, "peakRssMb": 13238, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 20316, "output": 662 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T22:59:25.391Z", "finishedAt": "2026-11-09T23:02:22.272Z", "timeToFirstArtifactMs": 67354, "timeToLastArtifactWriteMs": 172481, "timeToFirstTokenMs": 36064, "firstTurnTtftMs": 26436, "timeToFirstToolCallMs": 42077 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T22:59:31.224Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T22:59:31.141Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wren": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 8, "maxContextFill": 0.09259033203125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.09259033203125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-constrained-comms", "baselineCellId": "r1-saluki-cuda-productivity-constrained-comms", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "scenarioId": "constrained-comms", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-09T23:20:57.052Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/workspace/driftwater-outage-notice/customer-notice.md", "artifactBytes": 596, "artifactSha256": "d35246fa00bbeb22fe0e7e20040bc252044a41ec4ce580428537223d1b660062", "axisSource": "evals/src/scenarios/constrained-comms.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The notice gives the correct window, impact, remedy, payment-data disclosure and support address, but remains 92 words against a 140-word minimum after bounded repair. It adds unsupported account-access reassurance and implies nonrecurrence. This is an eligible task failure with no native load, OOM or context failure.", "reviewBasis": "Read complete declared artifact within20KiB and all relevant seeded source files, tool receipts, fixed facts and runtime events. Advisory only; artifacts, fixed scores and gates untouched.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/workspace/driftwater-outage-notice/customer-notice.md", "bytes": 596, "sha256": "d35246fa00bbeb22fe0e7e20040bc252044a41ec4ce580428537223d1b660062", "role": "declaredOutput", "status": "reviewed", "reviewedBytes": 596, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T22:59:33.218Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T22:59:36.266Z INFO [chat] [llama-server] launch {\"pid\":12844,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-09T23:02:21.489Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-12844-1794265341489\",\"pid\":12844,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":165224,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "nativeFailureEpisodes": 0, "explicitCudaOom": false, "maxObservedReleasedSlotTokens": 11254, "peakGpuMiB": 10640, "peakProcessRssMiB": 13238, "peakSystemRamMiB": 29505, "decodeTokensPerSec": 8.64, "prefillTokensPerSec": 577.5, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.09259033203125 }, "note": "Actual task placement is39/64dense FFNGPU+25CPU,65Kf16. Shared GPU memory/paging unmeasured. Nonzero native exit and memory-pressure cleanup labels alone do not diagnose CUDA OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:01.362Z", "reviewId": "098", "cellId": "r3-saluki-cuda-productivity-constrained-comms", "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "scenarioId": "constrained-comms", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "baselineCellId": "r1-saluki-cuda-productivity-constrained-comms", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "ded5778d8118649233b6604418cf4746616521fb8751494db13b668b254db5f2", "evidenceSha256": "21e6093a6fb5d597625413920cc3042964cb4d428d047deaeebc0124081d57d6", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/workspace/driftwater-outage-notice/customer-notice.md", "bytes": 596, "sha256": "d35246fa00bbeb22fe0e7e20040bc252044a41ec4ce580428537223d1b660062", "shownBytes": 596, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7\\workspace\\driftwater-outage-notice\\facts\\incident-brief.md", "bytes": 524, "sha256": "a65cd16d09c63d27126363a03f1bc664d26e4c6fc58eae192477d98ac3d02251", "shownBytes": 524, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7\\workspace\\driftwater-outage-notice\\facts\\legal-requirements.md", "bytes": 446, "sha256": "f15d48f9a7a37a2e3d855566ffd9cbdb9f0633a9f8c7f0756816b40a15815920", "shownBytes": 446, "truncated": false } ], "scoreAxes": {}, "justification": "The notice has the June 30 date, 38 minutes, the exact sentence, the support email, the one-month credit and no banned phrases. But I count about 88 words, well under the 140–220 hard limit. It also adds \"account access\" as unaffected, which the brief doesn't state.", "limitations": "The word count is my manual estimate from the artifact text. I didn't see the run transcript, so I can't tell whether the model checked length or read the stale draft. This is a source-only review with no execution.", "meanScore": null, "durationMs": 18719, "costUsd": 0.020384, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2415, "cache_read_input_tokens": 700, "output_tokens": 1065, "output_tokens_details": { "thinking_tokens": 789 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2415, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 1065, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2415, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2415 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 1065, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2415, "webSearchRequests": 0, "costUSD": 0.020384, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 789, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2415, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - constrained-comms / underdog-saluki-27b-iq2-mix\n\n**Composite: 6.0 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 2m 57s; budget used: 9.7%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`; `facts.timing.timeToFirstArtifactMs: 67354`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":594,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...` |\n| Output quality (25%) | 5 / 10 | An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 67354`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":594,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.097431959683403`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 67354`; `facts.outcome.durationMs: 176881` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **6.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13238 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29505 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.64 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 26235 + 776 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 9% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`\n- `facts.timing.timeToFirstArtifactMs: 67354`\n- `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":594,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n\n### Output quality - Q_PARTIAL_UNKNOWN_MAX\n\nAn authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 67354`\n- `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":594,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.097431959683403`\n- `facts.toolUse.totalToolCalls: 7`\n- `facts.timing.timeToFirstArtifactMs: 67354`\n- `facts.outcome.durationMs: 176881`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"repair-exhausted: bounded scenario feedback exhausted after 4 delivered-and-completed attempts on customer-notice.md; the same scenario failure stayed unchanged. Last failure: ...`; `facts.timing.timeToFirstArtifactMs: 67354`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":594,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n- Quality: An authored, scored output exists, but the gate denominator is unavailable; partial credit is capped at the rubric midpoint. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 67354`; `facts.sniff.latest: {\"filePath\":\"constrained-comms\",\"bytes\":594,\"score\":8,\"scoreMax\":null,\"signals\":[\"payment-data-disclosure\",\"support-contact\",\"service-credit\",\"no-guarantee\",\"no-never-again\",\"no...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.097431959683403`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 67354`; `facts.outcome.durationMs: 176881`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **6/10**, eligible task failure; unchanged. Advisory **notApplicable**, with no declared judge axes. No declared axes. The notice gives the correct window, impact, remedy, payment-data disclosure and support address, but remains 92 words against a 140-word minimum after bounded repair. It adds unsupported account-access reassurance and implies nonrecurrence. This is an eligible task failure with no native load, OOM or context failure.\n\n## Positives\n\nThe artifact satisfies the eight non-length signals: correct incident facts, payment-data disclosure, support contact, service credit and absence of the exact banned phrases (facts.sniff.latest.signals). It avoids the stale draft incorrect duration and uses calm, direct language. The task loads and writes normally at8.64decode tok/s; no native fault appears in this trial.\n\n## Negatives and limits\n\nThe first notice is80words, then88, then92; the last write leaves92words, so the standard bounded repair ends model-stuck after four completed attempts (facts.outcome.reason, facts.sniff.latest.failReason; log.txt). The 140-word floor is never met despite explicit feedback. The added phrase that account access was unaffected is absent from the source brief; the fix wording \"so this does not recur\" implies a stronger future assurance than the evidence supports. These are advisory content issues beyond the exact banned-phrase checks. Seven calls and low budget use do not erase the failure.\n\n## Performance and fit\n\nDecode 8.64tok/s; prefill 577.5tok/s. Peak GPU 10640/12227MiB, process RSS 13238MiB, system RAM 29505/31865MiB. Context 65536, 1 slot, f16; max observed released slot 11254. Native endpoint/nonzero-exit episodes: 0; no explicit CUDA OOM/context-overflow signature observed. Dense FFN placement is39GPU+25CPU; loading successfully is not fullGPU residency. Whole-system memory includes other applications; shared GPU memory/paging are unmeasured.\n\n## Tactical experiment — not applied\n\nExperiment only: in ../gilde/data/gezel-templates/co/copywriter/versions/1.1.0/about.md, add after Cut every sentence that is decorative: \"Honor every requested word minimum as well as the maximum. Count the whole deliverable, target the middle of the requested band, and add only source-supported customer-useful detail until it fits.\" The repeated112/106/92word misses across R1/R2/R3 support testing a generic length-aware workflow. Compare unrelated customer letters and bounded executive summaries; never inject a benchmark-specific target or boilerplate.\n\n## Strategic experiment — not applied\n\nExperiment only: expose a word-band result with current count, required minimum, maximum and deficit through packages/mcp/src/validate.ts using the existing core wordBand check. Pair this with a repair-progress diagnostic in packages/service/src/providers/unresolved-tool-failure-ledger.ts to distinguish content changes from actual criterion improvement. Keep factual grounding active so length expansion cannot add unsupported reassurance. No retry or gate change is applied here.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/facts.json): fail, fixed 6, 3.23min; eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/facts.json): fail, fixed 6, 8.00min; eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/facts.json): pass, fixed 10, 0.69min; eligible.\n\nUnrelated controls: original R2 codemod sweep passed8.5 while data-wrangle failed4.3. Target reusable dependency consistency, quantified constraints, source derivations and effective repair, not benchmark expected strings. The campaign makes no prompt or tuning change; original/continuation environment and automatic preflight scaling prevent causal quality or speed claims.\n\n## Recommended next experiment\n\nAfter the frozen campaign, evaluate the concrete tactical instruction on the unrelated task types named above, tracking acceptance, factuality, repair turns and advisory defects; retain current baseline scores.\n\n## Strategic note\n\nConstrained communications has three eligible fixed6 failures:112words inR1,106 inR2,92 inR3. That is a recurring floor-compliance and repair problem rather than a fit failure. The paired plan task repeatedly satisfies its upper bound, showing that obeying a ceiling does not establish the ability to meet a floor. Preserve all three failures; do not retry them.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-records-intake", "baselineCellId": "r1-saluki-cuda-productivity-records-intake", "suite": "productivity", "scenario": "records-intake", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-09T23:04:41.733Z", "finishedAt": "2026-11-10T00:20:38.304Z", "durationMs": 4556571, "success": false, "reason": "hit hard ceiling (2269289ms + 38m progress extensions) — last hard progress 228s before the end; the deliverable never closed", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "failureMode": "timeout", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "records-intake", "score": 0, "bytes": 0 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "timeout" }, { "source": "facts", "path": "outcome.reason", "value": "hit hard ceiling (2269289ms + 38m progress extensions) — last hard progress 228s before the end; the deliverable never closed" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1810375 }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "quality": { "score": 5, "ruleId": "Q_UNGRADED_AUTHORED_OUTPUT", "summary": "An authored output exists without a sniff verdict; quality is capped at the rubric midpoint.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1810375 }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 20 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1810375 }, { "source": "facts", "path": "outcome.durationMs", "value": 4556571 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.3, "band": "framework-gap" }, "facts": { "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "scenarioId": "records-intake", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 758, "peakRssMb": 12010, "peakCpuPercent": 331966.8, "firstAtMs": 1, "lastAtMs": 4539664 }, "gpu": { "available": true, "sampleCount": 758, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 758, "peakUsedMb": 28890, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 16757, "totalOutputTokens": 35208, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T00:20:28.388Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4541303, "meanTokensPerSec": 8.29, "genTokensPerSec": 8.29, "genTokens": 35208, "promptTokensPerSec": 372.17, "peakRssMb": 12010, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T23:04:47.145Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T23:04:47.049Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "griet": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0550537109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0550537109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "timeout", "reason": "hit hard ceiling (2269289ms + 38m progress extensions) — last hard progress 228s before the end; the deliverable never closed", "durationMs": 4556571, "timeoutMs": 2269289, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-09T23:04:41.733Z", "finishedAt": "2026-11-10T00:20:38.304Z", "timeToFirstArtifactMs": 1810375, "timeToLastArtifactWriteMs": 4316405, "timeToFirstTokenMs": 23901, "firstTurnTtftMs": 14653, "timeToFirstToolCallMs": 28952 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 20, "byTool": { "list_dir": 4, "read_file": 8, "read_artifact": 4, "search": 1, "write_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [], "latest": null }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-09T23:04:41.736Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "[2026-11-09T23:04:41.736Z] [trial] id=records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe scenario=records-intake model=underdog-saluki-27b-iq2-mix", "[2026-11-09T23:04:41.737Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-records-intake-nWNPUl", "[2026-11-09T23:04:41.737Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-09T23:04:41.737Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-09T23:04:41.739Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-09T23:04:41.744Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-09T23:04:41.749Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-records-intake-nWNPUl", "[2026-11-09T23:04:41.749Z] [trial] throughput-scaled ceiling: 25m → 38m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-09T23:04:47.101Z] [trial] daemon spawned pid=9840 port=https://127.0.0.1:56354", "[2026-11-09T23:04:48.047Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-09T23:04:48.063Z] [trial] running scenario setup hook", "[2026-11-09T23:04:48.063Z] [trial] meester=imara", "[2026-11-09T23:04:48.465Z] [trial] scenario setup complete", "[2026-11-09T23:04:48.466Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-09T23:04:48.540Z] [poll] initial digests hard=d61151702fd92644 soft=69a8185a3b7535db", "[2026-11-09T23:04:48.624Z] [poll] hard-progress: d61151702fd92644 → b743b3d1c1f69689 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-09T23:05:11.923Z] [poll] hard-progress: b743b3d1c1f69689 → f9a8242ad5ff7235 after 23s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=18,stream=29) sniff=0/0B)", "[2026-11-09T23:05:29.799Z] [poll] hard-progress: f9a8242ad5ff7235 → 271e297af908c23f after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=43,stream=93) sniff=0/0B)", "[2026-11-09T23:05:41.694Z] [poll] hard-progress: 271e297af908c23f → 164f740cccf6ab85 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=62,stream=155) sniff=0/0B)", "[2026-11-09T23:14:02.858Z] [poll] hard-progress: 164f740cccf6ab85 → 24eaa800aa04bf60 after 501s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=1436,stream=4212) sniff=0/0B)", "[2026-11-09T23:22:22.888Z] [poll] hard-progress: 24eaa800aa04bf60 → 7ab14669bf6f1be4 after 500s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=11,slot=2802,stream=8240) sniff=0/0B)", "[2026-11-09T23:30:41.073Z] [poll] hard-progress: 7ab14669bf6f1be4 → c9a58ec4c7f2cc53 after 498s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=12,slot=4153,stream=12218) sniff=0/0B)", "[2026-11-09T23:30:41.078Z] [poll] retry-loop nudge deferred: griet/02504060 in conference-registration-intake for 1553s, last tool call 2s ago is already working; refusing to queue a potentially stale edit", "[2026-11-09T23:30:53.025Z] [poll] hard-progress: c9a58ec4c7f2cc53 → 3745ad34fbb76da6 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=13,slot=4169,stream=12269) sniff=0/0B)" ] }, "metrics": { "process": { "sampleCount": 758, "peakRssMb": 12010, "peakCpuPercent": 331966.8, "firstAtMs": 1, "lastAtMs": 4539664 }, "gpu": { "available": true, "sampleCount": 758, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 758, "peakUsedMb": 28890, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 16757, "totalOutputTokens": 35208, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T00:20:28.388Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4541303, "meanTokensPerSec": 8.29, "genTokensPerSec": 8.29, "genTokens": 35208, "promptTokensPerSec": 372.17, "peakRssMb": 12010, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-09T23:04:41.733Z", "finishedAt": "2026-11-10T00:20:38.304Z", "timeToFirstArtifactMs": 1810375, "timeToLastArtifactWriteMs": 4316405, "timeToFirstTokenMs": 23901, "firstTurnTtftMs": 14653, "timeToFirstToolCallMs": 28952 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-09T23:04:47.145Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-09T23:04:47.049Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "griet": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0550537109375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0550537109375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-records-intake", "baselineCellId": "r1-saluki-cuda-productivity-records-intake", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "scenarioId": "records-intake", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-10T00:29:22.616Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/workspace/conference-registration-intake/records/attendees.csv", "artifactBytes": 0, "artifactSha256": null, "axisSource": "evals/src/scenarios/records-intake.ts (no declared axes; required output absent)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "The declared attendees.csv is absent. A 5218-byte helper was written three times but never executed; the logged 14-tool surface contains neither a script runner nor delegation. Static review also finds broken month parsing, reversed slash-date fields and lost emails. This is an eligible timeout, not a load/OOM/context failure.", "reviewBasis": "Confirmed required CSV is absent. Read the complete produced helper and four input/schema files within the20KiB cap; inspected tool receipts and exposed tool roster. Did not execute the helper or retry the trial.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/workspace/conference-registration-intake/records/attendees.csv", "role": "declaredOutput", "status": "noArtifact", "bytes": 0 }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/workspace/conference-registration-intake/scripts/consolidate-registrations.js", "role": "producedCompanion", "status": "notApplicable", "bytes": 5218, "sha256": "bcb3813d8b6feace059f0db8537ab105a4a5966abae7a673de3f0b7ff4aa03ee", "reviewedBytes": 5218, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-09T23:04:49.177Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-09T23:04:52.277Z INFO [chat] [llama-server] launch {\"pid\":18400,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T00:20:37.551Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-18400-1794270037551\",\"pid\":18400,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":4545275,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 22734, "peakGpuMiB": 10644, "peakProcessRssMiB": 12010, "peakSystemRamMiB": 28890, "decodeTokensPerSec": 8.29, "prefillTokensPerSec": 372.17, "generatedTokens": 35208, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0550537109375 }, "note": "Actual hybrid task placement:39/64 dense FFN layers GPU,25CPU,65536 context,one f16 slot. Shared GPU memory/paging unmeasured; whole-system memory includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:04.199Z", "reviewId": "099", "cellId": "r3-saluki-cuda-productivity-records-intake", "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "scenarioId": "records-intake", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "baselineCellId": "r1-saluki-cuda-productivity-records-intake", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "51b32c5c48e46bdf3ee903fc55742a723aff415348293708ece1e2205270f611", "evidenceSha256": "0b74efa7a1f14c0dca1dd863cf137832ff38cc1258bc6c09083d65165e4c20b9", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/workspace/conference-registration-intake/scripts/consolidate-registrations.js", "bytes": 5218, "sha256": "bcb3813d8b6feace059f0db8537ab105a4a5966abae7a673de3f0b7ff4aa03ee", "shownBytes": 5218, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "The required records/attendees.csv is absent. Only scripts/consolidate-registrations.js (5218 bytes, untruncated) exists. It would write that file, but no run, output or log is shown. The script has some questionable parsing, e.g. the MONTHS keys and the dd/mm vs mm/dd handling. Cause of the missing output is not shown.", "limitations": "No execution logs, transcript or source inputs were provided, so I can't tell whether the script ran, failed, or was never run. The run was runner-terminal, so no model success or failure is inferred. Script correctness was not verified, and no scores are assigned.", "meanScore": null, "durationMs": 14759, "costUsd": 0.020679999999999997, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4349, "cache_read_input_tokens": 700, "output_tokens": 321, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4349, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 321, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 4349, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4349 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 321, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 4349, "webSearchRequests": 0, "costUSD": 0.020679999999999997, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4349, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - records-intake / underdog-saluki-27b-iq2-mix\n\n**Composite: 3.3 / 10** (band: framework-gap)\n**Outcome: timeout** (duration: 75m 57s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (2269289ms + 38m progress extensions) — last hard progress 228s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 1810375`; `facts.sniff.latest: null` |\n| Output quality (25%) | 5 / 10 | An authored output exists without a sniff verdict; quality is capped at the rubric midpoint. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1810375`; `facts.sniff.latest: null` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 20`; `facts.timing.timeToFirstArtifactMs: 1810375`; `facts.outcome.durationMs: 4556571` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 5 + 0.2 * 2.5 + 0.15 * 10 = **3.3**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12010 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28890 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.29 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 16757 + 35208 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"timeout\"`\n- `facts.outcome.reason: \"hit hard ceiling (2269289ms + 38m progress extensions) — last hard progress 228s before the end; the deliverable never closed\"`\n- `facts.timing.timeToFirstArtifactMs: 1810375`\n- `facts.sniff.latest: null`\n\n### Output quality - Q_UNGRADED_AUTHORED_OUTPUT\n\nAn authored output exists without a sniff verdict; quality is capped at the rubric midpoint.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 1810375`\n- `facts.sniff.latest: null`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 20`\n- `facts.timing.timeToFirstArtifactMs: 1810375`\n- `facts.outcome.durationMs: 4556571`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (2269289ms + 38m progress extensions) — last hard progress 228s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 1810375`; `facts.sniff.latest: null`\n- Quality: An authored output exists without a sniff verdict; quality is capped at the rubric midpoint. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1810375`; `facts.sniff.latest: null`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 20`; `facts.timing.timeToFirstArtifactMs: 1810375`; `facts.outcome.durationMs: 4556571`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **3.3/10**, eligible timeout (standard failureClass=model), unchanged. Advisory **noArtifact**: records/attendees.csv does not exist. The produced helper is a separate companion with no declared judge axes, hence notApplicable; it is not the requested output.\n\n## Positives\n\nThe model read schema.md and all three source files (tool receipts; facts.toolUse.byTool.read_file=8 including later helper rereads), and wrote a5218-byte helper that attempts deduplication and CSV emission. Native inference stayed available for the full trial, with no compaction or explicit native/OOM/context fault (facts.continuity.compaction; daemon.log). These establish loading and tool access, not task completion.\n\n## Negatives and diagnosis\n\nAfter4556571ms (75.94minutes),20calls and35208generated tokens, the required CSV remains absent (facts.outcome, facts.sniff.latest=null). Four artifact reads guessed paths that did not exist. The script was written three times and reread four times, without any execution or delegation (facts.toolUse.byTool and tool receipts). The standard runner extended the initial37.82minute budget by38minutes because it observed progress, then honored its hard ceiling; no operator retry occurred.\n\nThe actual model-facing roster logged at23:04:50.838Z has14tools: filesystem reads/writes, validate, artifact/document reads and search. It does **not** expose run_nodejs_script, derive_file or a delegation tool, even though the bridge registers those tools and the task recommends producing derived data through execution. This is a role/tool mismatch for the chosen script approach. Direct CSV writing remains possible, so this is not a claim that the entire task was technically impossible. Keep the standard eligible task-failure classification while reporting this framework contribution separately.\n\nStatic helper review finds several independent bugs: parseDate looks up jun but the table stores june; the source day-first16/06/2026 becomes2026-16-06; upsert omits email while the emitter reads r.email; name matching fails the supplied comma/parenthesis/angle-bracket separators; and phone rows with spaced dates are skipped. The helper was not executed by the reviewer, and none of these counterfactual outputs is recorded as a fixed score.\n\nThe fixed quality5 uses Q_UNGRADED_AUTHORED_OUTPUT because a helper was authored. It does not mean attendees.csv was half correct. The fixed behavior10 reflects no registered red flags/missing roles; it does not establish that the role has every tool needed for its selected approach (score.json; facts.team.missingExpectedRoles=[]).\n\n## Performance and fit\n\nDecode 8.29tok/s, prefill 372.17tok/s. Peak GPU 10644/12227MiB, process RSS 12010MiB, whole-system RAM 28890/31865MiB. Context65536,one f16 slot,39dense FFN layersGPU+25CPU; maximum observed released slot 22734. No explicit native/OOM/context failure; shared GPU memory/paging remain unmeasured. The failure concerns delivery and tool selection, not demonstrated inability to load within the chosen hybrid allocation.\n\n## Tactical experiment — not applied\n\nIn ../gilde/data/gezel-templates/bo/boekwachter/versions/1.1.1/about.md, experiment with an explicit-user-job exception to the background-indexing identity: \"For an explicitly assigned deliverable, produce the named output; a helper is not a substitute. Check the available tools before choosing an execution-based approach. If required execution is unavailable, state the limitation instead of repeatedly rewriting or rereading a helper.\" This targets the observed no-execution loop (facts.toolUse), not a fixture name or expected row. Validate on unrelated inventories and document indexes.\n\n## Strategic experiment — not applied\n\nAudit bounded data-transformation capability assignment in packages/core/src/roles/registry.ts and packages/core/src/prompt/role-tool-filter.ts. For an authorized derived-file job, either expose the existing scoped derive_file/run_nodejs_script route under unchanged security policy or route to an appropriate worker before the job starts. Do not globally broaden the background Librarian permission set. Add tool-surface/deliverable compatibility diagnostics so a task cannot recommend execution that the final filtered roster lacks. Evidence is the actual14-tool roster, not merely absence of a call. Also measure declared-output progress separately from token generation and helper rewrites in evals/src/runner.ts before proposing different timeout behavior; retain all current gates and time budgets here.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/facts.json): pass, fixed 8.5, 108.38minutes; eligible.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/facts.json): fail, fixed 2, 52.34minutes; eligible.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/facts.json): pass, fixed 10, 2.16minutes; eligible.\n\nOriginal R1 failed2 with no output. R2 eventually passed8.5 with all15records after108minutes and an unused faulty helper; R3 again never emits the CSV. Unrelated R3 data-wrangle passes10 by actually executing its helper, whereas original R2 codemod sweep passes8.5. These comparisons point to execution/tool routing and declared-output tracking rather than a blanket model-size explanation. The original/continuation GEZEL_MXC_EXEC environment boundary and different automatic timeout scaling remain explicit.\n\n## Recommended next experiment\n\nAfter this campaign, test capability-aware routing on this workflow and an unrelated JSON transformation, measuring real output generation, source fidelity and time spent on unexecuted helpers without changing acceptance gates.\n\n## Strategic note\n\nThe recurring helper-without-output pattern now spans records-intake repetitions while data-wrangle demonstrates successful execution in the same model/native baseline. Preserve this distinction in the aggregate: successful hybrid loading and sustained8.29tok/s do not establish reliable task closure.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-plan-and-estimate", "baselineCellId": "r1-saluki-cuda-productivity-plan-and-estimate", "suite": "productivity", "scenario": "plan-and-estimate", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T00:23:29.235Z", "finishedAt": "2026-11-10T00:26:32.616Z", "durationMs": 183381, "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 178444 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "plan-and-estimate", "bytes": 3636, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.08080989243767542 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 2 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 178444 }, { "source": "facts", "path": "outcome.durationMs", "value": 183381 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "scenarioId": "plan-and-estimate", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 30, "peakRssMb": 9190, "peakCpuPercent": 1196.7, "firstAtMs": 0, "lastAtMs": 172058 }, "gpu": { "available": true, "sampleCount": 30, "peakUtilPercent": 95, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 26331, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4760, "totalOutputTokens": 1269, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T00:26:28.392Z" } }, "source": "engine-log" }, "derived": { "durationMs": 173592, "meanTokensPerSec": 8.33, "genTokensPerSec": 8.33, "genTokens": 1269, "promptTokensPerSec": 569.51, "peakRssMb": 9190, "peakGpuUtilPercent": 95, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T00:23:34.856Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T00:23:34.774Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.060028076171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.060028076171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "durationMs": 183381, "timeoutMs": 2269289, "budgetUsedFraction": 0.08080989243767542 }, "timing": { "startedAt": "2026-11-10T00:23:29.235Z", "finishedAt": "2026-11-10T00:26:32.616Z", "timeToFirstArtifactMs": 178444, "timeToLastArtifactWriteMs": 178444, "timeToFirstTokenMs": 24208, "firstTurnTtftMs": 14735, "timeToFirstToolCallMs": 31068 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 2, "byTool": { "read_files": 1, "write_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 178649, "filePath": "plan-and-estimate", "bytes": 3636, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } ], "latest": { "filePath": "plan-and-estimate", "bytes": 3636, "score": 6, "scoreMax": 6, "signals": [ "ordered-sections", "plan-structure", "ws-floor-plan", "ws-movers", "ws-network", "word-ceiling" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T00:23:29.238Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-4QmqLt", "[2026-11-10T00:23:29.238Z] [trial] id=plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o scenario=plan-and-estimate model=underdog-saluki-27b-iq2-mix", "[2026-11-10T00:23:29.238Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "[2026-11-10T00:23:29.238Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T00:23:29.238Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T00:23:29.241Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T00:23:29.244Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T00:23:29.251Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-plan-and-estimate-4QmqLt", "[2026-11-10T00:23:29.251Z] [trial] throughput-scaled ceiling: 25m → 38m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T00:23:34.753Z] [trial] daemon spawned pid=41712 port=https://127.0.0.1:57228", "[2026-11-10T00:23:35.640Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T00:23:35.656Z] [trial] meester=linnea", "[2026-11-10T00:23:35.656Z] [trial] running scenario setup hook", "[2026-11-10T00:23:36.032Z] [trial] scenario setup complete", "[2026-11-10T00:23:36.034Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T00:23:36.099Z] [poll] initial digests hard=98b44af4e9f2683a soft=2898fc19e4153f64", "[2026-11-10T00:23:36.170Z] [poll] hard-progress: 98b44af4e9f2683a → 84317c9921619f4c after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T00:24:05.318Z] [poll] hard-progress: 84317c9921619f4c → 17fc9ac3014bba5f after 29s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=31,stream=84) sniff=0/0B)", "[2026-11-10T00:26:27.884Z] [poll] terminal: success=true reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)", "[2026-11-10T00:26:28.426Z] [capture] dumped 1 session(s)", "[2026-11-10T00:26:28.428Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-10T00:26:28.489Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-10T00:26:32.615Z] [trial] daemon shut down", "[2026-11-10T00:26:32.625Z] [trial] PASS duration=183381ms reason=Plan satisfies structure + workstreams + ceiling (signals: ordered-sections, plan-structure, ws-floor-plan, ws-movers, ws-network, word-ceiling)" ] }, "metrics": { "process": { "sampleCount": 30, "peakRssMb": 9190, "peakCpuPercent": 1196.7, "firstAtMs": 0, "lastAtMs": 172058 }, "gpu": { "available": true, "sampleCount": 30, "peakUtilPercent": 95, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 30, "peakUsedMb": 26331, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 4760, "totalOutputTokens": 1269, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T00:26:28.392Z" } }, "source": "engine-log" }, "derived": { "durationMs": 173592, "meanTokensPerSec": 8.33, "genTokensPerSec": 8.33, "genTokens": 1269, "promptTokensPerSec": 569.51, "peakRssMb": 9190, "peakGpuUtilPercent": 95, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T00:23:29.235Z", "finishedAt": "2026-11-10T00:26:32.616Z", "timeToFirstArtifactMs": 178444, "timeToLastArtifactWriteMs": 178444, "timeToFirstTokenMs": 24208, "firstTurnTtftMs": 14735, "timeToFirstToolCallMs": 31068 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T00:23:34.856Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T00:23:34.774Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "deepak": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.060028076171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.060028076171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-plan-and-estimate", "baselineCellId": "r1-saluki-cuda-productivity-plan-and-estimate", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "scenarioId": "plan-and-estimate", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-10T00:33:16.153Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/workspace/harbourview-office-relocation/plan.md", "artifactBytes": 3640, "artifactSha256": "da76c8bbf2d9585ae26910a979680dc08cbfa39099574bae531713488d121a29", "axisSource": "evals/src/scenarios/plan-and-estimate.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. Ten tasks have valid internal owners, observable criteria and the required sign-off before mover booking; workshop completion is explicit. The plan only books movers and plans packing: no task executes packing, transport or unpacking, and the final check covers networking rather than the completed move.", "reviewBasis": "Read complete plan and seeded brief/team within20KiB, plus tool receipts, facts and native runtime events. Advisory only; no changed artifacts or fixed scoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/workspace/harbourview-office-relocation/plan.md", "role": "declaredOutput", "status": "notApplicable", "bytes": 3640, "sha256": "da76c8bbf2d9585ae26910a979680dc08cbfa39099574bae531713488d121a29", "reviewedBytes": 3640, "truncated": false } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T00:23:36.818Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T00:23:40.059Z INFO [chat] [llama-server] launch {\"pid\":40956,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T00:26:31.916Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-40956-1794270391916\",\"pid\":40956,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":171858,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 5966, "peakGpuMiB": 10638, "peakProcessRssMiB": 9190, "peakSystemRamMiB": 26331, "decodeTokensPerSec": 8.33, "prefillTokensPerSec": 569.51, "generatedTokens": 1269, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.060028076171875 }, "note": "Actual task allocation:65536 context,one f16 slot,39/64 dense FFN layers GPU and25CPU. Shared GPU memory/paging unmeasured; system memory includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:09.619Z", "reviewId": "100", "cellId": "r3-saluki-cuda-productivity-plan-and-estimate", "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "scenarioId": "plan-and-estimate", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "baselineCellId": "r1-saluki-cuda-productivity-plan-and-estimate", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "a618a0245b1162793edc1a401a5e449dc1b5f6e5934bdacdfef4cf07e6dfe0ee", "evidenceSha256": "bea7f0f964f146d90b8ed03405829b09f28d79e55eca39a2380e1f16a7c72573", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/workspace/harbourview-office-relocation/plan.md", "bytes": 3640, "sha256": "da76c8bbf2d9585ae26910a979680dc08cbfa39099574bae531713488d121a29", "shownBytes": 3640, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o\\workspace\\harbourview-office-relocation\\brief.md", "bytes": 732, "sha256": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "shownBytes": 732, "truncated": false } ], "scoreAxes": {}, "justification": "plan.md has the four sections in order, and the table header and 10 rows match the spec. Owners are all from the roster (Ola is not an owner), and the movers task T5 depends on sign-off T2. It covers the floor plan, movers and network workstreams. It is about 650 words, under the limit.", "limitations": "team.md was not shown, so the roster and the owner names could not be checked against it. The wording of \"T10 depends on T8, T9\" and the day-zero check is acceptable. The chat reply was not visible. There is no explicit date schedule or estimate, and the runner status is terminal with no numeric axes.", "meanScore": null, "durationMs": 14344, "costUsd": 0.0171, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3464, "cache_read_input_tokens": 700, "output_tokens": 317, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3464, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 317, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3464, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3464 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 317, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3464, "webSearchRequests": 0, "costUSD": 0.0171, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3464, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - plan-and-estimate / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 3m 3s; budget used: 8.1%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 178444`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3636,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.08080989243767542`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 178444`; `facts.outcome.durationMs: 183381` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9190 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 95% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26331 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.33 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 4760 + 1269 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 6% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 178444`\n- `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3636,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.08080989243767542`\n- `facts.toolUse.totalToolCalls: 2`\n- `facts.timing.timeToFirstArtifactMs: 178444`\n- `facts.outcome.durationMs: 183381`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 178444`; `facts.sniff.latest: {\"filePath\":\"plan-and-estimate\",\"bytes\":3636,\"score\":6,\"scoreMax\":6,\"signals\":[\"ordered-sections\",\"plan-structure\",\"ws-floor-plan\",\"ws-movers\",\"ws-network\",\"word-ceiling\"],\"fail...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.08080989243767542`; `facts.toolUse.totalToolCalls: 2`; `facts.timing.timeToFirstArtifactMs: 178444`; `facts.outcome.durationMs: 183381`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3 productivity\n\nFixed **10/10 pass**, unchanged. Advisory **notApplicable** because this scenario declares no judge axes. No declared axes. Ten tasks have valid internal owners, observable criteria and the required sign-off before mover booking; workshop completion is explicit. The plan only books movers and plans packing: no task executes packing, transport or unpacking, and the final check covers networking rather than the completed move.\n\n## Positives\n\nAll six frozen signals pass, including the three workstreams and word ceiling (facts.sniff.latest.score=6/6). The ten rows use internal owners; T5 booking depends on T2 formal approval. T4 defines a completed, inspected workshop rather than a mere agreed schedule. Two tool calls read the source files and write the plan in183381ms (facts.toolUse; facts.outcome.durationMs). Assumptions about external availability and network lead time are labeled as assumptions.\n\n## Negatives and limits\n\nThe top-level objective is a fully operational move, but T5 only books movers and T6 only publishes a packing/access plan. No row actually packs, transports or unpacks the studio; T10 checks only the network. The structural pass therefore does not establish complete operational coverage (facts.sniff.latest.signals; advisory-artifact-audit.json and plan.md). T7's completion criterion confirms an installation date, although the later live-web check does test the eventual connection. The source only explicitly gates mover booking on sign-off, so workshop preparation after the draft is not mislabeled as a hard-constraint violation. Do not convert these advisory observations into a new fixed score.\n\n## Performance and fit\n\nDecode 8.33tok/s; prefill 569.51tok/s; generated 1269tokens. Peak GPU 10638/12227MiB, process RSS 9190MiB, system RAM 26331/31865MiB. Actual context65536,one f16 slot,39dense FFN layers GPU+25CPU; max observed released slot 5966. One native launch and normal expected SIGTERM cleanup; no explicit native/OOM/context failure or compaction. This is hybrid loading, not all-GPU residency; shared GPU memory/paging are unmeasured.\n\n## Tactical experiment — not applied\n\nUnder Finish the whole feature path / Named deliverables in ../gilde/data/gezel-templates/de/developer/versions/1.5.1/about.md, test: \"Trace every promised objective to an execution task and final verification; booking or planning work is not the completed work.\" Evidence is the missing physical move execution despite facts.sniff.latest.score=6/6. Validate on unrelated onboarding and release plans, checking objective coverage and false-positive critiques without fixture-specific wording.\n\n## Strategic experiment — not applied\n\nMeasure an optional objective-to-task acceptance audit in packages/mcp/src/validate.ts: distinguish preparations from executed outcomes and check whether the overall objective has a closing acceptance task. Keep it advisory and test parallel plans with deliberately delegated execution so it does not require unnecessary steps. Do not change this campaign's frozen gate or source.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/facts.json): pass, fixed 10, 2.96minutes; baseline continuation round3.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/facts.json): pass, fixed 10, 3.18minutes; baseline original campaign.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/facts.json): pass, fixed 10, 3.53minutes; baseline original campaign.\n\nAll three preceding same-scenario attempts also score10. Earlier core R3 has nine rows and a scheduling-only workshop criterion; this productivity R3 has an explicit completed workshop, but omits actual move execution. Repetition across suites is not added task diversity, and prompt/native identity is unchanged within R3. Original history crosses the explicit GEZEL_MXC_EXEC environment boundary, so this is descriptive rather than a controlled improvement claim.\n\nUnrelated R3 data-wrangle passes10 after executing a transformation and producing the correct output; R3 records-intake fails3.3 with an unexecuted helper and no required CSV. Those outcomes support separately inspecting prepared work, executed work and final outputs across task types. They do not imply that this planning task should physically execute a real move; the plan should simply cover those execution steps.\n\n## Recommended next experiment\n\nAfter the frozen campaign, test the proposed objective-to-execution instruction on onboarding and release plans, measuring complete coverage, dependency validity and unnecessary extra work while preserving all present scores.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-meeting-followup", "baselineCellId": "r1-saluki-cuda-productivity-meeting-followup", "suite": "productivity", "scenario": "meeting-followup", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T00:29:08.900Z", "finishedAt": "2026-11-10T00:33:48.015Z", "durationMs": 279115, "success": true, "reason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 120022 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "meeting-followup", "bytes": 2046, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.10249722104609116 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 8 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 120022 }, { "source": "facts", "path": "outcome.durationMs", "value": 279115 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "scenarioId": "meeting-followup", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 46, "peakRssMb": 12280, "peakCpuPercent": 1216.2, "firstAtMs": 0, "lastAtMs": 267180 }, "gpu": { "available": true, "sampleCount": 46, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 46, "peakUsedMb": 29021, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 23579, "totalOutputTokens": 1570, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 6232, "todayTokensCached": 0, "todayTokensOut": 1074, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 6232, "totalTokensCached": 0, "totalTokensOut": 1074, "totalCost": 0, "medianOutputTokensPerSec": 8.6, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.6, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3467, "outputTokens": 537, "cost": 0, "durationMs": 105550, "outputTokensPerSec": 8.529025403224791, "at": "2026-11-10T00:32:54.482Z" }, "lastUpdated": "2026-11-10T00:32:54.482Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T00:33:43.597Z" } }, "source": "engine-log" }, "derived": { "durationMs": 268710, "meanTokensPerSec": 8.56, "genTokensPerSec": 8.56, "genTokens": 1570, "promptTokensPerSec": 397.14, "peakRssMb": 12280, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6232, "output": 1074 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T00:29:14.917Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T00:29:14.801Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "ashot": 1, "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 3, "maxContextFill": 0.094146728515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.094146728515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "durationMs": 279115, "timeoutMs": 2723147, "budgetUsedFraction": 0.10249722104609116 }, "timing": { "startedAt": "2026-11-10T00:29:08.900Z", "finishedAt": "2026-11-10T00:33:48.015Z", "timeToFirstArtifactMs": 120022, "timeToLastArtifactWriteMs": 269045, "timeToFirstTokenMs": 36319, "firstTurnTtftMs": 26501, "timeToFirstToolCallMs": 49872 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 8, "byTool": { "read_file": 4, "write_file": 3, "list_dir": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 274176, "filePath": "meeting-followup", "bytes": 2046, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } ], "latest": { "filePath": "meeting-followup", "bytes": 2046, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "launch-decision", "migration-decision", "annual-plan-open", "rollback-owner-open", "legal-risk", "stale-proposals-rejected", "csv-header", "four-actions", "action-row-a1", "action-row-a2", "action-row-a3", "action-row-a4" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T00:29:08.904Z] [trial] id=meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3 scenario=meeting-followup model=underdog-saluki-27b-iq2-mix", "[2026-11-10T00:29:08.904Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "[2026-11-10T00:29:08.904Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T00:29:08.904Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-meeting-followup-osAiC3", "[2026-11-10T00:29:08.904Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T00:29:08.907Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T00:29:08.911Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T00:29:08.918Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-meeting-followup-osAiC3", "[2026-11-10T00:29:08.918Z] [trial] throughput-scaled ceiling: 30m → 45m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T00:29:14.830Z] [trial] daemon spawned pid=44328 port=https://127.0.0.1:58712", "[2026-11-10T00:29:15.728Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T00:29:15.746Z] [trial] running scenario setup hook", "[2026-11-10T00:29:15.746Z] [trial] meester=jasper", "[2026-11-10T00:29:16.214Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T00:29:16.212Z] [trial] scenario setup complete", "[2026-11-10T00:29:16.293Z] [poll] initial digests hard=1b3e36a46d2debf2 soft=eeefe3fb69ed52ef", "[2026-11-10T00:29:16.370Z] [poll] hard-progress: 1b3e36a46d2debf2 → e7ac85e255201440 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T00:30:03.171Z] [poll] hard-progress: e7ac85e255201440 → e94f37148047c88a after 47s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=37,stream=82) sniff=0/0B)", "[2026-11-10T00:30:56.858Z] [poll] hard-progress: e94f37148047c88a → 193db00874c85ece after 54s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=3,slot=180,stream=889) sniff=0/0B)", "[2026-11-10T00:31:14.558Z] [poll] hard-progress: 193db00874c85ece → f93db33536938c2e after 18s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=215,stream=1078) sniff=1/1739B)", "[2026-11-10T00:31:38.308Z] [poll] hard-progress: f93db33536938c2e → bab251f47c7f5b9d after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=261,stream=1198) sniff=1/1739B)", "[2026-11-10T00:31:44.161Z] [poll] hard-progress: bab251f47c7f5b9d → c0d34655538b996b after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=267,stream=1215) sniff=1/1739B)", "[2026-11-10T00:32:55.348Z] [poll] hard-progress: c0d34655538b996b → 6e4813ce677ce963 after 71s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=445,stream=2211) sniff=1/1739B)", "[2026-11-10T00:33:43.077Z] [poll] terminal: success=true reason=meeting brief and action register pass all reconciliation gates (ordered-sections, launch-decision, migration-decision, annual-plan-open, rollback-owner-open, legal-risk, stale-proposals-rejected, csv-header, four-actions, action-row-a1, action-row-a2, action-row-a3, action-row-a4)", "[2026-11-10T00:33:43.650Z] [capture] dumped 2 session(s)" ] }, "metrics": { "process": { "sampleCount": 46, "peakRssMb": 12280, "peakCpuPercent": 1216.2, "firstAtMs": 0, "lastAtMs": 267180 }, "gpu": { "available": true, "sampleCount": 46, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 46, "peakUsedMb": 29021, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 23579, "totalOutputTokens": 1570, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 6232, "todayTokensCached": 0, "todayTokensOut": 1074, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 6232, "totalTokensCached": 0, "totalTokensOut": 1074, "totalCost": 0, "medianOutputTokensPerSec": 8.6, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.6, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 3467, "outputTokens": 537, "cost": 0, "durationMs": 105550, "outputTokensPerSec": 8.529025403224791, "at": "2026-11-10T00:32:54.482Z" }, "lastUpdated": "2026-11-10T00:32:54.482Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T00:33:43.597Z" } }, "source": "engine-log" }, "derived": { "durationMs": 268710, "meanTokensPerSec": 8.56, "genTokensPerSec": 8.56, "genTokens": 1570, "promptTokensPerSec": 397.14, "peakRssMb": 12280, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 6232, "output": 1074 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T00:29:08.900Z", "finishedAt": "2026-11-10T00:33:48.015Z", "timeToFirstArtifactMs": 120022, "timeToLastArtifactWriteMs": 269045, "timeToFirstTokenMs": 36319, "firstTurnTtftMs": 26501, "timeToFirstToolCallMs": 49872 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T00:29:14.917Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T00:29:14.801Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "ashot": 1, "noor": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 3, "maxContextFill": 0.094146728515625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.094146728515625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-meeting-followup", "baselineCellId": "r1-saluki-cuda-productivity-meeting-followup", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "scenarioId": "meeting-followup", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T01:27:00.586Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/meeting-brief.md", "artifactBytes": 1745, "artifactSha256": "9fb585f34d26808576410c40a738bad999f9eeefc09437739fb09b47475f677c", "axisSource": "evals/src/scenarios/meeting-followup.ts judge.axes", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ], "scoreAxes": { "decisionClarity": 7, "grounding": 8, "actionability": 8, "brevity": 7 }, "axisJustifications": { "decisionClarity": "Current decisions and unresolved issues are distinct; scheduling the annual-plan review is also listed under Decisions despite being an action.", "grounding": "Dates, owners, EU-only scope, overnight migration and the A1 dependency match the source. Scheduling by August30 is preserved without turning it into a decision deadline.", "actionability": "The four-row action table gives owners, deadlines, dependency and status, matching the companion CSV. Legal and rollback gaps remain visible without invented assignments.", "brevity": "Headings and tables are easy to scan, but stale proposals and annual-plan scheduling recur in the introduction, decisions and open questions." }, "meanScore": 7.5, "justification": "Accurate decisions, owners, dates and the A1 dependency; annual-plan and rollback questions stay open, and the four-row CSV matches. The scheduling deadline is preserved correctly. Scheduling the review also appears under Decisions, and repeated stale-agenda caveats add modest redundancy. Clear and usable, with minor organization and brevity issues.", "reviewBasis": "Read complete declared brief and companion CSV within20KiB each, all three seeded sources, tool receipts, facts and native runtime events. Exact scenario axes and frozen0-10 rubric; advisory only.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/meeting-brief.md", "role": "declaredJudgeArtifact", "status": "scored", "bytes": 1745, "sha256": "9fb585f34d26808576410c40a738bad999f9eeefc09437739fb09b47475f677c", "reviewedBytes": 1745, "truncated": false }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/action-items.csv", "role": "requiredCompanion", "status": "notApplicable", "bytes": 307, "sha256": "2c081c4b560ff6b6fd8ef6f24f9d4f6031888754ec558fd15adea0802d6ca31c", "reviewedBytes": 307, "truncated": false, "note": "No separate declared axes for CSV; all four rows independently reconciled with the transcript and brief." } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T00:29:16.916Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T00:29:19.991Z INFO [chat] [llama-server] launch {\"pid\":15032,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T00:33:47.286Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-15032-1794270827286\",\"pid\":15032,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":267296,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 7606, "peakGpuMiB": 10640, "peakProcessRssMiB": 12280, "peakSystemRamMiB": 29021, "decodeTokensPerSec": 8.56, "prefillTokensPerSec": 397.14, "generatedTokens": 1570, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.094146728515625 }, "note": "Actual task allocation:65536 context,one f16 slot,39/64 dense FFN layers GPU and25CPU. Shared GPU memory/paging unmeasured; system memory includes other applications." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:18.516Z", "reviewId": "101", "cellId": "r3-saluki-cuda-productivity-meeting-followup", "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "scenarioId": "meeting-followup", "status": "scored", "axes": [ { "name": "decisionClarity", "description": "Separates decisions, open questions, actions, and risks cleanly." }, { "name": "grounding", "description": "Owners, dates, scope, and dependencies match the transcript and roster." }, { "name": "actionability", "description": "A reader can execute the follow-up without rereading the transcript." }, { "name": "brevity", "description": "The brief is scannable and avoids transcript-style repetition." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "baselineCellId": "r1-saluki-cuda-productivity-meeting-followup", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "7aee9ecd7c72de354b4835fe3e00142a61b1f3ce7085784c230adb50121eb463", "evidenceSha256": "7e71eac0d72536195efb7ba710876832f6c063cc3fc7faaddeaacf0019685f10", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/meeting-brief.md", "bytes": 1745, "sha256": "9fb585f34d26808576410c40a738bad999f9eeefc09437739fb09b47475f677c", "shownBytes": 1745, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/action-items.csv", "bytes": 307, "sha256": "2c081c4b560ff6b6fd8ef6f24f9d4f6031888754ec558fd15adea0802d6ca31c", "shownBytes": 307, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3\\workspace\\orchid-launch-follow-up\\meeting\\old-agenda.md", "bytes": 262, "sha256": "8a895418b408627cf5fd7367a45341bedc5fc29515c61dac05de3b47f6fdde38", "shownBytes": 262, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3\\workspace\\orchid-launch-follow-up\\meeting\\roster.md", "bytes": 223, "sha256": "8f77db20d7700521aecdf6cddcbbe807ebf99bae5c2043a558fd5f96831cad69", "shownBytes": 223, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3\\workspace\\orchid-launch-follow-up\\meeting\\transcript.md", "bytes": 977, "sha256": "7a4039bda8b643bfdfb3f8d1208a5c7835fd3b13f280867db9b6915992e0800a", "shownBytes": 977, "truncated": false } ], "scoreAxes": { "decisionClarity": 8, "grounding": 9, "actionability": 8, "brevity": 8 }, "justification": "H2 order is correct. The stale agenda's proposals are explicitly marked as non-decisions. The annual plan and rollback-communications owner stay open. The CSV header and four rows match the brief, with full roster names, todo status, and empty depends_on where nothing is a dependency. Minor blemish: the A4 scheduling commitment is listed under Decisions.", "limitations": "Source-only review of the shown files. The files were not opened in a rendered view, and CSV parsing was judged by eye. Whether the file locations match the workspace-root requirement could not be checked from the shown paths.", "meanScore": 8.25, "durationMs": 17083, "costUsd": 0.0218166, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3832, "cache_read_input_tokens": 846, "output_tokens": 640, "output_tokens_details": { "thinking_tokens": 299 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3832, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 640, "cache_read_input_tokens": 846, "cache_creation_input_tokens": 3832, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3832 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 640, "cacheReadInputTokens": 846, "cacheCreationInputTokens": 3832, "webSearchRequests": 0, "costUSD": 0.0218166, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 299, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3832, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - meeting-followup / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 4m 39s; budget used: 10.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 120022`; `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2046,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.10249722104609116`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 120022`; `facts.outcome.durationMs: 279115` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12280 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29021 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.56 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 23579 + 1570 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 9% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 120022`\n- `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2046,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.10249722104609116`\n- `facts.toolUse.totalToolCalls: 8`\n- `facts.timing.timeToFirstArtifactMs: 120022`\n- `facts.outcome.durationMs: 279115`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 120022`; `facts.sniff.latest: {\"filePath\":\"meeting-followup\",\"bytes\":2046,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"launch-decision\",\"migration-decision\",\"annual-plan-open\",\"rollback-owner-open...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.10249722104609116`; `facts.toolUse.totalToolCalls: 8`; `facts.timing.timeToFirstArtifactMs: 120022`; `facts.outcome.durationMs: 279115`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **10/10 pass**, unchanged. Inline advisory **7.5/10**, separate from the capability composite. The entire brief (1745UTF8bytes) and required CSV (307bytes) were reviewed without truncation. The brief declares four axes; the companion CSV has no separate axes and is notApplicable for its own score.\n\n| Declared axis | Advisory score | Evidence |\n|---|---:|---|\n| decisionClarity | 7 | Current decisions and unresolved issues are distinct; scheduling the annual-plan review is also listed under Decisions despite being an action. |\n| grounding | 8 | Dates, owners, EU-only scope, overnight migration and the A1 dependency match the source. Scheduling by August30 is preserved without turning it into a decision deadline. |\n| actionability | 8 | The four-row action table gives owners, deadlines, dependency and status, matching the companion CSV. Legal and rollback gaps remain visible without invented assignments. |\n| brevity | 7 | Headings and tables are easy to scan, but stale proposals and annual-plan scheduling recur in the introduction, decisions and open questions. |\n\nAccurate decisions, owners, dates and the A1 dependency; annual-plan and rollback questions stay open, and the four-row CSV matches. The scheduling deadline is preserved correctly. Scheduling the review also appears under Decisions, and repeated stale-agenda caveats add modest redundancy. Clear and usable, with minor organization and brevity issues.\n\n## Positives\n\nAll13 reconciliation signals pass (facts.sniff.latest.score=13/13). Independent source review confirms September14,EU-only,one overnight batch; exact four owners/actions/dates and only A2 depending on A1; and the unassigned rollback/legal-review gaps. The review scheduling deadline remains a scheduling deadline, not an invented decision due date. Eight calls produce both requested files in279115ms (facts.toolUse; facts.outcome.durationMs). There are no fixed red flags or missing expected roles (facts.toolUse.redFlags; facts.team.missingExpectedRoles).\n\n## Negatives and process limits\n\nThe review-scheduling action appears both under Decisions and in the action table, and stale-agenda disclaimers repeat across sections (advisory-artifact-audit.json; facts.sniff.latest confirms the relevant content signals, not prose economy). This is a modest presentation issue, not a fixed gate failure.\n\nThe standard missing-deliverable mechanism ensured Developer Ashot at00:30:56Z while Planner Noor was still producing the brief. Noor wrote it at00:31:08Z; Ashot listed/read it, then rewrote identical content at00:32:54Z. A second nudge targeted the CSV, which Ashot wrote at00:33:37Z. Identical write arguments verified: true. This is redundant work across actors, not evidence of a malformed tool call or a capability retry (facts.toolUse.byTool.write_file=3; facts.team.rolesCreated; advisory-tool-evidence.json). facts.autoAnswer.total=0 measures user-answer interventions and must not be read as zero harness nudges.\n\n## Performance and fit\n\nDecode 8.56tok/s; prefill 397.14tok/s; generated 1570tokens. Peak GPU 10640/12227MiB, process RSS 12280MiB, system RAM 29021/31865MiB. Context65536,one f16 slot,39dense FFN layers GPU+25CPU; max observed released slot 7606. One launch with normal expected SIGTERM cleanup, no explicit native/OOM/context failure and no compaction. Shared GPU memory/paging are unmeasured; this remains hybrid inference.\n\n## Tactical experiment — not applied\n\nUnder Working style in ../gilde/data/gezel-templates/pl/planner/versions/1.1.0/about.md, test: \"Keep agreed decisions, execution actions and unresolved choices distinct. Mention superseded alternatives once unless repeating them resolves a specific ambiguity.\" Evidence is the repeated scheduling action and agenda caveats despite a clean13/13 gate. Measure clarity and retained uncertainty on unrelated incident summaries and project handoffs; do not bake names, dates or benchmark headers into the role.\n\n## Strategic experiment — not applied\n\nInspect evals/src/sniff-feedback.ts missing-deliverable routing: measure whether an active, authorized writer can finish before an extra implementation specialist is created, and have a replacement specialist recheck which named outputs remain missing before rewriting an already valid file. Evidence is the12-second overlap before Noor's brief and Ashot's identical rewrite. Compare document-plus-register and report-plus-data tasks; preserve genuine stall rescue and all security boundaries. No routing, prompts, gates or timeout policy changed in this campaign.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/facts.json): pass, fixed 10, 5.02minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/facts.json): pass, fixed 10, 13.57minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/facts.json): pass, fixed 10, 2.83minutes; other historical baseline; contextual only.\n\nOriginal R1 advisory7.5 and R2 advisory7.0 accompany their preserved fixed scores. R3 advisory7.5 avoids R2's scheduling-deadline drift but retains modest repetition. Original/continuation GEZEL_MXC_EXEC and automatic timing boundaries prevent causal attribution. Other-model history is contextual, not a recommendation.\n\nUnrelated R3 constrained-comms fails6 on a word minimum and records-intake fails3.3 with no required CSV; meeting-followup demonstrates accurate paired prose/CSV output in the same current baseline. Core conflict-synthesis passes8.5 after repeated repair and a recovered endpoint episode. These differences support measuring exact constraints, completion and orchestration separately rather than generalizing from successful loading.\n\n## Recommended next experiment\n\nAfter the frozen campaign, test active-writer-aware rescue on a document/register task and an unrelated report/data task, measuring duplicate writes, completed deliverables and time to recovery when the original writer truly stalls.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-craftbook-spreadsheet-model", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-spreadsheet-model", "suite": "productivity", "scenario": "craftbook-spreadsheet-model", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T00:36:23.882Z", "finishedAt": "2026-11-10T02:07:04.455Z", "durationMs": 5440573, "success": false, "reason": "hit hard ceiling (2723147ms + 45m progress extensions) — last hard progress 825s before the end; the deliverable never closed", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "failureMode": "timeout", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-spreadsheet-model", "score": 4, "bytes": 3629, "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md", "milestones": 3 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "timeout" }, { "source": "facts", "path": "outcome.reason", "value": "hit hard ceiling (2723147ms + 45m progress extensions) — last hard progress 825s before the end; the deliverable never closed" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 246181 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-spreadsheet-model", "bytes": 3629, "score": 4, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md" } } ] }, "quality": { "score": 6.7, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 66.7% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 246181 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-spreadsheet-model", "bytes": 3629, "score": 4, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 18 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 246181 }, { "source": "facts", "path": "outcome.durationMs", "value": 5440573 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.9, "band": "framework-gap" }, "facts": { "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "scenarioId": "craftbook-spreadsheet-model", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 905, "peakRssMb": 14448, "peakCpuPercent": 1211.5, "firstAtMs": 0, "lastAtMs": 5423791 }, "gpu": { "available": true, "sampleCount": 905, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 905, "peakUsedMb": 30656, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 49079, "totalOutputTokens": 38668, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 28726, "todayTokensCached": 20492, "todayTokensOut": 1934, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 28726, "totalTokensCached": 20492, "totalTokensOut": 1934, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 20849, "cachedInputTokens": 20492, "outputTokens": 63, "cost": 0, "durationMs": 3058137, "outputTokensPerSec": 8.882200814526467, "at": "2026-11-10T01:31:34.693Z" }, "lastUpdated": "2026-11-10T01:31:34.693Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T02:06:54.745Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5425383, "meanTokensPerSec": 8.75, "genTokensPerSec": 8.75, "genTokens": 38668, "promptTokensPerSec": 384.22, "peakRssMb": 14448, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28726, "output": 1934 } } } }, "craftbook": { "craftbookId": "spreadsheet-model", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 2, "gateApprovals": 2, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T00:36:30.772Z", "completedAt": "2026-11-10T00:40:30.212Z", "ms": 239440, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "f7354b0a-add7-40ed-9378-202b5c16ebe2" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "build", "activatedAt": "2026-11-10T00:40:30.213Z", "completedAt": "2026-11-10T01:31:34.668Z", "ms": 3064455, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "dd08f2d2-901a-499e-b1a5-a0e5f7a0e811" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-10T01:31:34.669Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "02236fc5-37ce-4ee3-be70-c8af4dc153ce" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T00:36:29.427Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T00:36:29.329Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1651947.5 }, "sessions": { "total": 3, "taskScoped": 3, "byGezel": { "eden": 1, "milagros": 1, "mirella": 1 }, "perTask": { "spreadsheet-financial-model-workflow-eval/1": 3 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6, "maxMessages": 2, "maxContextFill": 0.1259765625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1259765625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 3, "citations": [ "knowledge://bendyline/handboek/model-scorecard", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/ops-runbook" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "timeout", "reason": "hit hard ceiling (2723147ms + 45m progress extensions) — last hard progress 825s before the end; the deliverable never closed", "durationMs": 5440573, "timeoutMs": 2723147, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-10T00:36:23.882Z", "finishedAt": "2026-11-10T02:07:04.455Z", "timeToFirstArtifactMs": 246181, "timeToLastArtifactWriteMs": 3310812, "timeToFirstTokenMs": 33475, "firstTurnTtftMs": 20204, "timeToFirstToolCallMs": 246149 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Planner", "Developer", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 18, "byTool": { "write_task_note": 4, "write_artifact": 1, "read_artifact": 2, "read_file": 7, "write_file": 1, "replace_in_file": 1, "advance_task_step": 1, "validate": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/spreadsheet-financial-model-workflow-eval/index.html", "finalBytes": 19470, "growth": [] } ], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 7114, "filePath": "craftbook-spreadsheet-model", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, source/brief.md, source/records.csv" }, { "atMs": 249706, "filePath": "craftbook-spreadsheet-model", "bytes": 3629, "score": 2, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, source/brief.md, source/records.csv" }, { "atMs": 297854, "filePath": "craftbook-spreadsheet-model", "bytes": 3629, "score": 2, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, source/records.csv" }, { "atMs": 321617, "filePath": "craftbook-spreadsheet-model", "bytes": 3629, "score": 2, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md" }, { "atMs": 1554192, "filePath": "craftbook-spreadsheet-model", "bytes": 3629, "score": 4, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md" } ], "latest": { "filePath": "craftbook-spreadsheet-model", "bytes": 3629, "score": 4, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T00:36:23.885Z] [trial] id=craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc scenario=craftbook-spreadsheet-model model=underdog-saluki-27b-iq2-mix", "[2026-11-10T00:36:23.885Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "[2026-11-10T00:36:23.886Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T00:36:23.886Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-spreadsheet-model-YGkqo4", "[2026-11-10T00:36:23.888Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T00:36:23.886Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T00:36:23.892Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T00:36:23.898Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-spreadsheet-model-YGkqo4", "[2026-11-10T00:36:23.898Z] [trial] throughput-scaled ceiling: 30m → 45m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T00:36:29.395Z] [trial] daemon spawned pid=42312 port=https://127.0.0.1:61998", "[2026-11-10T00:36:30.304Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T00:36:30.320Z] [trial] meester=thandiwe", "[2026-11-10T00:36:30.320Z] [trial] running scenario setup hook", "[2026-11-10T00:36:30.911Z] [trial] scenario setup complete", "[2026-11-10T00:36:30.913Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T00:36:30.957Z] [poll] initial digests hard=8f7aed10b5e1e3f2 soft=81e313196fa4e1d7", "[2026-11-10T00:36:31.034Z] [poll] hard-progress: 8f7aed10b5e1e3f2 → b207e8e2001afe8d after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T00:36:36.197Z] [poll] hard-progress: b207e8e2001afe8d → 087b669805aee647 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T00:40:33.635Z] [poll] hard-progress: 087b669805aee647 → 2a63a01e86d7ec44 after 237s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=610,stream=2944) sniff=2/3629B)", "[2026-11-10T00:40:39.492Z] [poll] hard-progress: 2a63a01e86d7ec44 → 731f22ba08885a73 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=610,stream=2944) sniff=2/3629B)", "[2026-11-10T00:41:21.792Z] [poll] hard-progress: 731f22ba08885a73 → f922990f5f9ffc17 after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=652,stream=3046) sniff=2/3629B)", "[2026-11-10T00:41:45.570Z] [poll] hard-progress: f922990f5f9ffc17 → fa234585c2fa7b2e after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=716,stream=3237) sniff=2/3629B)", "[2026-11-10T01:02:18.149Z] [poll] hard-progress: fa234585c2fa7b2e → 39406eee78c5d81f after 1233s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=4271,stream=19811) sniff=4/3629B)", "[2026-11-10T01:02:30.101Z] [poll] hard-progress: 39406eee78c5d81f → d245d8f6b3f426aa after 12s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=4287,stream=19857) sniff=4/3629B)", "[2026-11-10T01:09:35.853Z] [poll] hard-progress: d245d8f6b3f426aa → 08f5e35d6e78c96e after 426s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=5468,stream=23465) sniff=4/3629B)" ] }, "metrics": { "process": { "sampleCount": 905, "peakRssMb": 14448, "peakCpuPercent": 1211.5, "firstAtMs": 0, "lastAtMs": 5423791 }, "gpu": { "available": true, "sampleCount": 905, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 905, "peakUsedMb": 30656, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 49079, "totalOutputTokens": 38668, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 28726, "todayTokensCached": 20492, "todayTokensOut": 1934, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 28726, "totalTokensCached": 20492, "totalTokensOut": 1934, "totalCost": 0, "medianOutputTokensPerSec": 8.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.8, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 20849, "cachedInputTokens": 20492, "outputTokens": 63, "cost": 0, "durationMs": 3058137, "outputTokensPerSec": 8.882200814526467, "at": "2026-11-10T01:31:34.693Z" }, "lastUpdated": "2026-11-10T01:31:34.693Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T02:06:54.745Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5425383, "meanTokensPerSec": 8.75, "genTokensPerSec": 8.75, "genTokens": 38668, "promptTokensPerSec": 384.22, "peakRssMb": 14448, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28726, "output": 1934 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T00:36:23.882Z", "finishedAt": "2026-11-10T02:07:04.455Z", "timeToFirstArtifactMs": 246181, "timeToLastArtifactWriteMs": 3310812, "timeToFirstTokenMs": 33475, "firstTurnTtftMs": 20204, "timeToFirstToolCallMs": 246149 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 2, "gateApprovals": 2, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T00:36:30.772Z", "completedAt": "2026-11-10T00:40:30.212Z", "ms": 239440, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "f7354b0a-add7-40ed-9378-202b5c16ebe2" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "build", "activatedAt": "2026-11-10T00:40:30.213Z", "completedAt": "2026-11-10T01:31:34.668Z", "ms": 3064455, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "dd08f2d2-901a-499e-b1a5-a0e5f7a0e811" ] }, { "ref": "spreadsheet-financial-model-workflow-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-10T01:31:34.669Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "02236fc5-37ce-4ee3-be70-c8af4dc153ce" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T00:36:29.427Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T00:36:29.329Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1651947.5 }, "sessions": { "total": 3, "taskScoped": 3, "byGezel": { "eden": 1, "milagros": 1, "mirella": 1 }, "perTask": { "spreadsheet-financial-model-workflow-eval/1": 3 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6, "maxMessages": 2, "maxContextFill": 0.1259765625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1259765625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-craftbook-spreadsheet-model", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-spreadsheet-model", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "scenarioId": "craftbook-spreadsheet-model", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T16:51:42.689Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "artifactBytes": 3665, "artifactSha256": "18638d25f76746720335eed029b4f6a1c215a3f6ea6a1f16d57f9d29773dc6ca", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/sp/spreadsheet-model/versions/1.0.5/test.json rubric, wired by evals/src/craftbooks/scenario.ts", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": { "workflow fidelity": 5, "usefulness": 4 }, "meanScore": 4.5, "justification": "The scope defines inputs, formulas and a correct zero-growth check; real scope/build steps completed. The SaaS assumptions are weakly tied to the supplied business. Required brief reading and verdict closure are missing. The calculator has correct headline math but blank calculation/output tables and a runtime exception, missed by an all-PASS task note.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "status": "scored", "bytes": 3665, "sha256": "18638d25f76746720335eed029b4f6a1c215a3f6ea6a1f16d57f9d29773dc6ca", "reviewedBytes": 3665, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/workspace/spreadsheet-financial-model-workflow-eval/index.html", "status": "notApplicable", "bytes": 19470, "sha256": "465c2c064f2d7298945ea39b8847befaabbf585dd25024a3ecd270b6925d31c0", "reviewedBytes": 19470, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/verdict.md", "status": "noArtifact", "bytes": 0, "role": "requiredWorkflowOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T00:36:35.381Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T00:36:38.451Z INFO [chat] [llama-server] launch {\"pid\":4464,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T02:07:03.718Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-4464-1794276423718\",\"pid\":4464,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":5425268,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 24874, "peakGpuMiB": 10644, "peakProcessRssMiB": 14448, "peakSystemRamMiB": 30656, "decodeTokensPerSec": 8.75, "prefillTokensPerSec": 384.22, "generatedTokens": 38668, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1259765625 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:24.449Z", "reviewId": "102", "cellId": "r3-saluki-cuda-productivity-craftbook-spreadsheet-model", "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "scenarioId": "craftbook-spreadsheet-model", "status": "scored", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-spreadsheet-model", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "4fe54a81b889e2c16dac95dc10285308eb7bb37255c3bde2f849877feb527540", "evidenceSha256": "ab899d9622d1029c70c1695eb2e6316266b2ec763e3d53f1f78a81b6ce4e1e49", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md", "bytes": 3665, "sha256": "18638d25f76746720335eed029b4f6a1c215a3f6ea6a1f16d57f9d29773dc6ca", "shownBytes": 3665, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/workspace/spreadsheet-financial-model-workflow-eval/index.html", "bytes": 19470, "sha256": "465c2c064f2d7298945ea39b8847befaabbf585dd25024a3ecd270b6925d31c0", "shownBytes": 19470, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\brief.md", "bytes": 791, "sha256": "9b7893b96d2e07d87b152dcdef1c8e16395615b8bc827a34cc10dd0706ab0bda", "shownBytes": 791, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\craftbook-eval-brief.md", "bytes": 2268, "sha256": "6cebbc3d1462d96e4ee669842d5d1d5fcc2c9e0c95bf6e723aefa688b297cbdd", "shownBytes": 2268, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc\\workspace\\spreadsheet-financial-model-workflow-eval\\source\\records.csv", "bytes": 141, "sha256": "430befe636c28a4b2751e842a79773440af0bd7a5f1b742ea177c6ef204d74c6", "shownBytes": 141, "truncated": false } ], "scoreAxes": { "workflow fidelity": 6, "usefulness": 5 }, "justification": "scope.md sets purpose, 8 drivers, formulas, acceptance checklist and a hand-checkable sanity case before the build, and index.html follows it with three zones. The evaluate verdict.md is missing. In index.html, paint() is called with a null formatter for out-break-even, which throws a TypeError and leaves Zone 3 unfilled. The \"12-Month Total\" row sums unrelated line items.", "limitations": "Source-only review, nothing was run. The null-formatter crash is inferred from reading the code. The scope states runway as about month 11, but cash is still positive at month 11 (20,000) and goes negative in month 12. The brief's Boreal Desk facts are unused.", "meanScore": 5.5, "durationMs": 20167, "costUsd": 0.0745438, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 15988, "cache_read_input_tokens": 778, "output_tokens": 1051, "output_tokens_details": { "thinking_tokens": 688 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 15988, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 1051, "cache_read_input_tokens": 778, "cache_creation_input_tokens": 15988, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 15988 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 1051, "cacheReadInputTokens": 778, "cacheCreationInputTokens": 15988, "webSearchRequests": 0, "costUSD": 0.0745438, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 688, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 15988, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-spreadsheet-model / underdog-saluki-27b-iq2-mix\n\n**Composite: 4.9 / 10** (band: framework-gap)\n**Outcome: timeout** (duration: 90m 41s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (2723147ms + 45m progress extensions) — last hard progress 825s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 246181`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3629,\"score\":4,\"scoreMax\":6,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-...` |\n| Output quality (25%) | 6.7 / 10 | The latest scenario gate reached 66.7% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 246181`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3629,\"score\":4,\"scoreMax\":6,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 18`; `facts.timing.timeToFirstArtifactMs: 246181`; `facts.outcome.durationMs: 5440573` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 6.7 + 0.2 * 2.5 + 0.15 * 10 = **4.9**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14448 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30656 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.75 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 49079 + 38668 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 2 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1652s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 3 / 0.60 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 13% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 3 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"timeout\"`\n- `facts.outcome.reason: \"hit hard ceiling (2723147ms + 45m progress extensions) — last hard progress 825s before the end; the deliverable never closed\"`\n- `facts.timing.timeToFirstArtifactMs: 246181`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3629,\"score\":4,\"scoreMax\":6,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 66.7% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 246181`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3629,\"score\":4,\"scoreMax\":6,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 18`\n- `facts.timing.timeToFirstArtifactMs: 246181`\n- `facts.outcome.durationMs: 5440573`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (2723147ms + 45m progress extensions) — last hard progress 825s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 246181`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3629,\"score\":4,\"scoreMax\":6,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-...`\n- Quality: The latest scenario gate reached 66.7% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 246181`; `facts.sniff.latest: {\"filePath\":\"craftbook-spreadsheet-model\",\"bytes\":3629,\"score\":4,\"scoreMax\":6,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 18`; `facts.timing.timeToFirstArtifactMs: 246181`; `facts.outcome.durationMs: 5440573`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **4.9/10**, eligible task failure; unchanged. Advisory **4.5/10**. The scope defines inputs, formulas and a correct zero-growth check; real scope/build steps completed. The SaaS assumptions are weakly tied to the supplied business. Required brief reading and verdict closure are missing. The calculator has correct headline math but blank calculation/output tables and a runtime exception, missed by an all-PASS task note.\n\nExact declared axes: workflow fidelity=5; usefulness=4.\n\n## Positives\n\nThe actual craftbook completed scope and build and entered evaluate (facts.continuity.steps.completed=2). The scope contains eight labeled assumptions, formula contracts and a correct zero-growth example. Browser review independently confirmed the default headline math: month12 net -134552.09, ending cash74406.04 and total net-1925593.96; zero-growth and high preset also match a separate closed-form calculation (advisory-ui-checks-20261110/checks.json; facts.sniff.latest.score=4/6). No native/OOM/context fault or compaction was observed during90.68minutes of sustained8.75tok/s inference.\n\n## Negatives and diagnosis\n\nThe required source/craftbook-eval-brief.md was never read, and evaluate never produced tasks/1/verdict.md, so the standard runner timed out after45minutes of progress extensions (facts.sniff.latest.failReason; facts.outcome; facts.continuity.steps). The planner selected a generic SaaS model before reading the provided home-office business fixtures; the values are labeled assumptions, not sourced client facts. The reviewer wrote an all-PASS task note at01:42:11Z, then continued reading until cutoff, without committing the required verdict. This is not an infrastructure exclusion or a capability retry.\n\nActual reviewer validate reported runtime-load skipped: browser-runtime-not-installed. The exposed roster and craftbook policy prohibit browser automation/code execution; static syntax PASS does not demonstrate interactivity. Advisory Chromium review of the exact saved HTML finds all156 calculation cells blank: buildTable puts data-row on td while render reads it from tr. Every render also throws \"fmt is not a function\" when a null formatter is passed for the runway label, leaving the runway and entire output table blank. Three headline cards and the sanity calculation do work. Mobile document width is1035 at390px, and the8 numeric inputs have no associated label/aria-label. A latent total-row formula also adds overlapping revenue/cost/profit categories rather than a meaningful12-month aggregate; this branch is currently bypassed by the missing row ID.\n\nThe original builder note and reviewer arithmetic match the independent calculation; the builder later \"correction\" to -134210.63/74634.59 is wrong. Do not conflate correct arithmetic with the broken rendering. The reviewer claim of exercised flow-through has no browser tool receipt. The trial screenshot recorder reported ERR_FILE_NOT_FOUND although captured index.html exists; advisory review used setContent of those exact bytes with HTTP(S) blocked. This review neither modifies artifacts nor changes fixed4.9.\n\n## Performance and fit\n\nDecode 8.75tok/s, prefill 384.22tok/s; 38668generated tokens. Peak GPU 10644/12227MiB, process RSS 14448MiB, whole-system RAM 30656/31865MiB. Context 65536, 1slot, f16; max observed released slot 24874. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn ../gilde/data/gezel-templates/re/reviewer/versions/1.4.1/about.md, strengthen the existing evidence caveat: \"When runtime validation is skipped, explicitly leave interactive behavior unverified; a calculation or author note is not evidence that the displayed cells update. Deliver the requested verdict file, not only a task note.\" Measure false PASS claims and missing verdicts on unrelated forms and chart dashboards; no scenario-specific expected output.\n\n## Strategic experiment — not applied\n\nEvaluate a constrained headless interaction/check route through packages/service/src/http/routes/page-check.ts and packages/mcp/src/validate.ts, with explicit availability and observation receipts. The current craftbook asks for interactive exercise while its evaluate toolPolicy disables browser automation and code execution. Resolve that contract in a future version of ../gilde/data/craftbook-templates/sp/spreadsheet-model/versions/1.0.5/craftbook.json and the shared workflow authoring policy, rather than granting broad execution. Also measure verdict-file progress separately from repeated reads/notes in evals/src/runner.ts. Preserve the current baseline/gates.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/facts.json): pass, fixed 8, 74.96minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/facts.json): fail, fixed 4.5, 77.74minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/facts.json): pass, fixed 9.5, 2.46minutes; other historical baseline; contextual only.\n\nR1 spreadsheet timed out without a calculator; R2 completed but had scenario actual cells stuck at zero. R3 produces a calculator yet again fails to close and falsely claims interactive acceptance. This recurring display-versus-internal-math gap is distinct from GPU fit. Unrelated R3 data-wrangle executes its transformation and passes10, while records-intake fails3.3 with an unexecuted helper. Generic runtime evidence and final-deliverable tracking are appropriate experiments.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nAfter the campaign, test constrained runtime observation and explicit unverified verdicts on a calculator and unrelated form, measuring blank/unchanged UI detection, false PASS claims and final verdict delivery without changing acceptance criteria.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-conflict-synthesis", "baselineCellId": "r1-saluki-cuda-productivity-conflict-synthesis", "suite": "productivity", "scenario": "conflict-synthesis", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T02:09:59.229Z", "finishedAt": "2026-11-10T02:40:36.782Z", "durationMs": 1837553, "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1121128 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "conflict-synthesis", "bytes": 2423, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6747902335055728 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 23 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1121128 }, { "source": "facts", "path": "outcome.durationMs", "value": 1837553 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" }, "facts": { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "scenarioId": "conflict-synthesis", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 305, "peakRssMb": 14615, "peakCpuPercent": 175841.1, "firstAtMs": 1, "lastAtMs": 1824579 }, "gpu": { "available": true, "sampleCount": 305, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 305, "peakUsedMb": 30824, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 43115, "totalOutputTokens": 4304, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 28140, "todayTokensCached": 17765, "todayTokensOut": 1276, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 28140, "totalTokensCached": 17765, "totalTokensOut": 1276, "totalCost": 0, "medianOutputTokensPerSec": 2.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.9, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8197, "cachedInputTokens": 4772, "outputTokens": 154, "cost": 0, "durationMs": 199277, "outputTokensPerSec": 2.854809749880602, "at": "2026-11-10T02:40:26.023Z" }, "lastUpdated": "2026-11-10T02:40:26.023Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T02:40:31.735Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1826048, "meanTokensPerSec": 2.82, "genTokensPerSec": 2.82, "genTokens": 4304, "promptTokensPerSec": 174.33, "peakRssMb": 14615, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28140, "output": 1276 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T02:10:05.462Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T02:10:05.352Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 6, "maxContextFill": 0.1057281494140625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1057281494140625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "all three conflicts reconciled explicitly with citations (signals: ordered-sections, conflict-launch-date-surfaced, conflict-launch-date-quarantined, conflict-launch-date-canonical-used, conflict-budget-surfaced, conflict-budget-quarantined, conflict-budget-canonical-used, conflict-dri-surfaced, conflict-dri-quarantined, conflict-dri-canonical-used, claims-source-supported, source-reads-grounded, citations)", "durationMs": 1837553, "timeoutMs": 2723147, "budgetUsedFraction": 0.6747902335055728 }, "timing": { "startedAt": "2026-11-10T02:09:59.229Z", "finishedAt": "2026-11-10T02:40:36.782Z", "timeToFirstArtifactMs": 1121128, "timeToLastArtifactWriteMs": 1826772, "timeToFirstTokenMs": 89658, "firstTurnTtftMs": 77879, "timeToFirstToolCallMs": 151652 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 23, "byTool": { "read_file": 19, "write_file": 3, "replace_in_file": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 1121829, "filePath": "conflict-synthesis", "bytes": 2809, "score": 11, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry" }, { "atMs": 1259119, "filePath": "conflict-synthesis", "bytes": 2809, "score": 12, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry" }, { "atMs": 1627366, "filePath": "conflict-synthesis", "bytes": 2433, "score": 12, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": "- conflict (launch budget): a superseded value appears outside the Conflicts section; re-read memo-product.md and finance.csv, then keep the historical value only in its reconciliation entry" }, { "atMs": 1831969, "filePath": "conflict-synthesis", "bytes": 2423, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } ], "latest": { "filePath": "conflict-synthesis", "bytes": 2423, "score": 13, "scoreMax": 13, "signals": [ "ordered-sections", "conflict-launch-date-surfaced", "conflict-launch-date-quarantined", "conflict-launch-date-canonical-used", "conflict-budget-surfaced", "conflict-budget-quarantined", "conflict-budget-canonical-used", "conflict-dri-surfaced", "conflict-dri-quarantined", "conflict-dri-canonical-used", "claims-source-supported", "source-reads-grounded", "citations" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T02:09:59.233Z] [trial] id=conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu scenario=conflict-synthesis model=underdog-saluki-27b-iq2-mix", "[2026-11-10T02:09:59.234Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "[2026-11-10T02:09:59.234Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-kGv8zR", "[2026-11-10T02:09:59.234Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T02:09:59.234Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T02:09:59.236Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T02:09:59.240Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T02:09:59.248Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-conflict-synthesis-kGv8zR", "[2026-11-10T02:09:59.248Z] [trial] throughput-scaled ceiling: 30m → 45m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T02:10:05.350Z] [trial] daemon spawned pid=18900 port=https://127.0.0.1:57084", "[2026-11-10T02:10:06.699Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T02:10:06.717Z] [trial] meester=fenton", "[2026-11-10T02:10:06.717Z] [trial] running scenario setup hook", "[2026-11-10T02:10:07.209Z] [trial] scenario setup complete", "[2026-11-10T02:10:07.211Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T02:10:07.338Z] [poll] initial digests hard=cbfe2bc23fd3a450 soft=29279d59c8f3d15b", "[2026-11-10T02:10:07.417Z] [poll] hard-progress: cbfe2bc23fd3a450 → 9a7b837741b6b41c after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T02:12:35.942Z] [poll] hard-progress: 9a7b837741b6b41c → c780add363a22dcb after 149s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=63,stream=94) sniff=0/0B)", "[2026-11-10T02:22:05.983Z] [poll] hard-progress: c780add363a22dcb → 6122ea67ba0b03da after 570s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=1560,stream=2428) sniff=0/0B)", "[2026-11-10T02:22:48.144Z] [poll] hard-progress: 6122ea67ba0b03da → dc8585534cc7ffe4 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=1573,stream=2442) sniff=0/0B)", "[2026-11-10T02:28:41.204Z] [poll] hard-progress: dc8585534cc7ffe4 → 8440a107d9a380b0 after 353s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=2478,stream=4172) sniff=11/2809B)", "[2026-11-10T02:29:04.751Z] [poll] hard-progress: 8440a107d9a380b0 → 75439a076611896a after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=10,slot=2501,stream=4202) sniff=11/2809B)", "[2026-11-10T02:30:58.395Z] [poll] hard-progress: 75439a076611896a → a3afc1fd183d28bb after 114s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=10,slot=2777,stream=4496) sniff=12/2809B)", "[2026-11-10T02:31:42.288Z] [poll] hard-progress: a3afc1fd183d28bb → dfad514f481320cf after 44s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=15,slot=2815,stream=4568) sniff=12/2809B)", "[2026-11-10T02:37:06.740Z] [poll] hard-progress: dfad514f481320cf → 286023a60d45ead0 after 324s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=3,tools=16,slot=3604,stream=6131) sniff=12/2433B)" ] }, "metrics": { "process": { "sampleCount": 305, "peakRssMb": 14615, "peakCpuPercent": 175841.1, "firstAtMs": 1, "lastAtMs": 1824579 }, "gpu": { "available": true, "sampleCount": 305, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 305, "peakUsedMb": 30824, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 43115, "totalOutputTokens": 4304, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 28140, "todayTokensCached": 17765, "todayTokensOut": 1276, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 28140, "totalTokensCached": 17765, "totalTokensOut": 1276, "totalCost": 0, "medianOutputTokensPerSec": 2.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.9, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8197, "cachedInputTokens": 4772, "outputTokens": 154, "cost": 0, "durationMs": 199277, "outputTokensPerSec": 2.854809749880602, "at": "2026-11-10T02:40:26.023Z" }, "lastUpdated": "2026-11-10T02:40:26.023Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T02:40:31.735Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1826048, "meanTokensPerSec": 2.82, "genTokensPerSec": 2.82, "genTokens": 4304, "promptTokensPerSec": 174.33, "peakRssMb": 14615, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28140, "output": 1276 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T02:09:59.229Z", "finishedAt": "2026-11-10T02:40:36.782Z", "timeToFirstArtifactMs": 1121128, "timeToLastArtifactWriteMs": 1826772, "timeToFirstTokenMs": 89658, "firstTurnTtftMs": 77879, "timeToFirstToolCallMs": 151652 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T02:10:05.462Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T02:10:05.352Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "tamsin": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 6, "maxContextFill": 0.1057281494140625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1057281494140625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-conflict-synthesis", "baselineCellId": "r1-saluki-cuda-productivity-conflict-synthesis", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "scenarioId": "conflict-synthesis", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-10T16:57:45.920Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/workspace/skylark-launch-synthesis/synthesis.md", "artifactBytes": 2433, "artifactSha256": "73a2f8bfa43227b1ea5da0f4991edd34049ce94a558b97ab36660a7dc3856a15", "axisSource": "evals/src/scenarios/conflict-synthesis.ts (no declared axes)", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "Canonical launch date, budget and DRI are reconciled with citations to all six read sources. Historical cadence is cautiously left unresolved. The correct final artifact followed repeated full-file reads and rewrites. No declared subjective axes.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/workspace/skylark-launch-synthesis/synthesis.md", "status": "notApplicable", "bytes": 2433, "sha256": "73a2f8bfa43227b1ea5da0f4991edd34049ce94a558b97ab36660a7dc3856a15", "reviewedBytes": 2433, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T02:10:09.140Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T02:10:13.417Z INFO [chat] [llama-server] launch {\"pid\":23332,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T02:40:35.998Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-23332-1794278435997\",\"pid\":23332,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":1822581,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 14354, "peakGpuMiB": 10642, "peakProcessRssMiB": 14615, "peakSystemRamMiB": 30824, "decodeTokensPerSec": 2.82, "prefillTokensPerSec": 174.33, "generatedTokens": 4304, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1057281494140625 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:23.942Z", "reviewId": "103", "cellId": "r3-saluki-cuda-productivity-conflict-synthesis", "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "scenarioId": "conflict-synthesis", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "baselineCellId": "r1-saluki-cuda-productivity-conflict-synthesis", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "2e6e590bcb407f690bf427d857dc36b211cfa1ee8f86efee03872bcdd3ed92d7", "evidenceSha256": "83df007b310824f39c0ef4701208ab2cc57a3547207e23486555317e83a9de37", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/workspace/skylark-launch-synthesis/synthesis.md", "bytes": 2433, "sha256": "73a2f8bfa43227b1ea5da0f4991edd34049ce94a558b97ab36660a7dc3856a15", "shownBytes": 2433, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "The synthesis has the four required sections in order. It records three conflicts (date, budget, DRI) with both values and file names, plus a winner and reason. The plan uses only winning values and cites five files. One weakness: the Overview says scope follows the product memo.", "limitations": "The source files were not shown, so the quoted values and any other conflicts could not be checked. The sync-cadence item is a speculative conflict. The scope claim and the onboarding revamp mention are unverified. No axes were declared, so nothing is scored.", "meanScore": null, "durationMs": 14230, "costUsd": 0.013710000000000002, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2694, "cache_read_input_tokens": 700, "output_tokens": 286, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2694, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 286, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2694, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2694 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 286, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2694, "webSearchRequests": 0, "costUSD": 0.013710000000000002, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2694, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - conflict-synthesis / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 30m 38s; budget used: 67.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1121128`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2423,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.6747902335055728`; `facts.toolUse.totalToolCalls: 23`; `facts.timing.timeToFirstArtifactMs: 1121128`; `facts.outcome.durationMs: 1837553` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 5 + 0.15 * 10 = **9.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14615 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30824 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.82 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 43115 + 4304 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 11% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1121128`\n- `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2423,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.6747902335055728`\n- `facts.toolUse.totalToolCalls: 23`\n- `facts.timing.timeToFirstArtifactMs: 1121128`\n- `facts.outcome.durationMs: 1837553`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1121128`; `facts.sniff.latest: {\"filePath\":\"conflict-synthesis\",\"bytes\":2423,\"score\":13,\"scoreMax\":13,\"signals\":[\"ordered-sections\",\"conflict-launch-date-surfaced\",\"conflict-launch-date-quarantined\",\"conflict...`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.6747902335055728`; `facts.toolUse.totalToolCalls: 23`; `facts.timing.timeToFirstArtifactMs: 1121128`; `facts.outcome.durationMs: 1837553`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **9/10**, pass; unchanged. Advisory **notApplicable**. Canonical launch date, budget and DRI are reconciled with citations to all six read sources. Historical cadence is cautiously left unresolved. The correct final artifact followed repeated full-file reads and rewrites. No declared subjective axes.\n\n\n\n## Positives\n\nThe synthesis separates three contradictory source values from the canonical September 1 launch, EUR210000 budget and Marcus ownership. It supplies explicit precedence and useful open questions without inventing missing budget allocations.\n\n## Negatives and diagnosis\n\nNineteen read calls and four edits are excessive for six short inputs. The cadence is described as superseded/unknown even though supersession is not established; the subsequent question appropriately avoids treating it as current fact. Decode is2.82tok/s versus the earlier round3 core trial; power, paging and shared GPU memory were not measured, so the slowdown has no established cause.\n\n## Performance and fit\n\nDecode 2.82tok/s, prefill 174.33tok/s; 4304generated tokens. Peak GPU 10642/12227MiB, process RSS 14615MiB, whole-system RAM 30824/31865MiB. Context 65536, 1slot, f16; max observed released slot 14354. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing research/synthesis role, repair only the challenged claim and reread the affected paragraph, retaining the already read source ledger. Test on conflicting budgets and schedules without adding Skylark-specific text.\n\n## Strategic experiment — not applied\n\nExpose provenance-aware changed-section validation so local models can reconcile evidence without repeatedly consuming the entire source set. Evaluate on incident and meeting synthesis too.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/facts.json): pass, fixed 8.5, 28.30minutes; continuation round3.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/facts.json): pass, fixed 10, 7.20minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/facts.json): pass, fixed 8.5, 18.84minutes; original campaign environment.\n\nThe same scenario has both core and productivity placements. Earlier coreR3 incurred a recovered native exit and a false-positive gate on a correct budget difference; this final artifact cleanly reconciles the canonical facts. Counts alone do not establish better source judgment. Unrelated data-wrangle passed with independently correct calculations, whereas records-intake produced no CSV.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nCompare a generic citation-preserving targeted-repair instruction across conflict-synthesis, incident-brief and meeting-notes, holding fixed rubric and native settings constant.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-docblocks-theme-roundtrip", "baselineCellId": "r1-saluki-cuda-productivity-docblocks-theme-roundtrip", "suite": "productivity", "scenario": "docblocks-theme-roundtrip", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T02:43:21.397Z", "finishedAt": "2026-11-10T03:02:34.953Z", "durationMs": 1153556, "success": true, "reason": "theme report is tool-grounded and the saved deck carries the brand (ordered-sections, theme-id, fonts, palette, slide-count, compare-cited, differences-not-whitewashed, recommendation, docblocks-provenance)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "nativeEngineIncidents": { "count": 1, "kinds": { "unexpected-exit": 1 }, "incidentIds": [ "native-39960-1794279259941" ], "evidence": [ "unexpected-exit" ] }, "engineContext": { "launches": 2, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1099981 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "docblocks-theme", "bytes": 1847, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.4236113584760573 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1099981 }, { "source": "facts", "path": "outcome.durationMs", "value": 1153556 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "scenarioId": "docblocks-theme-roundtrip", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 191, "peakRssMb": 11355, "peakCpuPercent": 1200.1, "firstAtMs": 1, "lastAtMs": 1138320 }, "gpu": { "available": true, "sampleCount": 191, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 191, "peakUsedMb": 28384, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 53449, "totalOutputTokens": 1623, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 28165, "todayTokensCached": 28142, "todayTokensOut": 95, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 28165, "totalTokensCached": 28142, "totalTokensOut": 95, "totalCost": 0, "medianOutputTokensPerSec": 2.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.5, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 28165, "cachedInputTokens": 28142, "outputTokens": 95, "cost": 0, "durationMs": 483206, "outputTokensPerSec": 2.5196674114070867, "at": "2026-11-10T03:02:21.989Z" }, "lastUpdated": "2026-11-10T03:02:21.989Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T03:02:27.640Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1139747, "meanTokensPerSec": 2.57, "genTokensPerSec": 2.57, "genTokens": 1623, "promptTokensPerSec": 118.45, "peakRssMb": 11355, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28165, "output": 95 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T02:43:27.667Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T02:43:27.540Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wieke": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.3077392578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3077392578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "nativeEngineIncidents": { "count": 1, "kinds": { "unexpected-exit": 1 }, "incidentIds": [ "native-39960-1794279259941" ], "evidence": [ "unexpected-exit" ] }, "outcome": { "success": true, "reason": "theme report is tool-grounded and the saved deck carries the brand (ordered-sections, theme-id, fonts, palette, slide-count, compare-cited, differences-not-whitewashed, recommendation, docblocks-provenance)", "durationMs": 1153556, "timeoutMs": 2723147, "budgetUsedFraction": 0.4236113584760573 }, "timing": { "startedAt": "2026-11-10T02:43:21.397Z", "finishedAt": "2026-11-10T03:02:34.953Z", "timeToFirstArtifactMs": 1099981, "timeToLastArtifactWriteMs": 1140593, "timeToFirstTokenMs": 182927, "firstTurnTtftMs": 167220, "timeToFirstToolCallMs": 196412 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Copywriter" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "infer_theme_from_file": 1, "apply_inferred_theme": 1, "convert_document": 1, "list_roots": 1, "save_artifact": 1, "inspect_document": 1, "compare_documents": 1, "write_file": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 1145733, "filePath": "docblocks-theme", "bytes": 1847, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } ], "latest": { "filePath": "docblocks-theme", "bytes": 1847, "score": 9, "scoreMax": 9, "signals": [ "ordered-sections", "theme-id", "fonts", "palette", "slide-count", "compare-cited", "differences-not-whitewashed", "recommendation", "docblocks-provenance" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T02:43:21.402Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "[2026-11-10T02:43:21.401Z] [trial] id=docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls scenario=docblocks-theme-roundtrip model=underdog-saluki-27b-iq2-mix", "[2026-11-10T02:43:21.402Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T02:43:21.402Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-docblocks-theme-roundtrip-MHRb3g", "[2026-11-10T02:43:21.405Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T02:43:21.402Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T02:43:21.409Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T02:43:21.415Z] [trial] throughput-scaled ceiling: 30m → 45m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T02:43:21.415Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-docblocks-theme-roundtrip-MHRb3g", "[2026-11-10T02:43:27.629Z] [trial] daemon spawned pid=30364 port=https://127.0.0.1:52825", "[2026-11-10T02:43:28.770Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T02:43:28.787Z] [trial] meester=elara", "[2026-11-10T02:43:28.787Z] [trial] running scenario setup hook", "[2026-11-10T02:43:29.288Z] [trial] scenario setup complete", "[2026-11-10T02:43:29.291Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T02:43:29.389Z] [poll] initial digests hard=260a7a9e38686b16 soft=4790aa1303a24705", "[2026-11-10T02:43:29.555Z] [poll] hard-progress: 260a7a9e38686b16 → 0c62c92e1cdccf9c after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T02:46:40.929Z] [poll] hard-progress: 0c62c92e1cdccf9c → 36112ec3f307efb3 after 191s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=40,stream=49) sniff=0/0B)", "[2026-11-10T02:47:16.530Z] [poll] hard-progress: 36112ec3f307efb3 → 59c0285d75ad18b7 after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=82,stream=129) sniff=0/0B)", "[2026-11-10T02:48:04.766Z] [poll] hard-progress: 59c0285d75ad18b7 → 4d2fc3073de6324f after 48s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=158,stream=275) sniff=0/0B)", "[2026-11-10T02:48:22.795Z] [poll] hard-progress: 4d2fc3073de6324f → f6dd013542f64396 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=164,stream=281) sniff=0/0B)", "[2026-11-10T02:49:12.081Z] [poll] hard-progress: f6dd013542f64396 → d273e7aebf972f56 after 49s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=248,stream=445) sniff=0/0B)", "[2026-11-10T02:49:54.398Z] [poll] hard-progress: d273e7aebf972f56 → 701e2fbc6e859264 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=301,stream=545) sniff=0/0B)", "[2026-11-10T02:50:47.864Z] [poll] hard-progress: 701e2fbc6e859264 → c2e37086994749aa after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=373,stream=681) sniff=0/0B)", "[2026-11-10T02:54:16.210Z] [poll] hard-progress: c2e37086994749aa → 95eeff34cc4043de after 208s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=836,stream=1601) sniff=0/0B)" ] }, "metrics": { "process": { "sampleCount": 191, "peakRssMb": 11355, "peakCpuPercent": 1200.1, "firstAtMs": 1, "lastAtMs": 1138320 }, "gpu": { "available": true, "sampleCount": 191, "peakUtilPercent": 100, "peakMemUsedMb": 10632, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 191, "peakUsedMb": 28384, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 53449, "totalOutputTokens": 1623, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 28165, "todayTokensCached": 28142, "todayTokensOut": 95, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 28165, "totalTokensCached": 28142, "totalTokensOut": 95, "totalCost": 0, "medianOutputTokensPerSec": 2.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.5, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 28165, "cachedInputTokens": 28142, "outputTokens": 95, "cost": 0, "durationMs": 483206, "outputTokensPerSec": 2.5196674114070867, "at": "2026-11-10T03:02:21.989Z" }, "lastUpdated": "2026-11-10T03:02:21.989Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T03:02:27.640Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1139747, "meanTokensPerSec": 2.57, "genTokensPerSec": 2.57, "genTokens": 1623, "promptTokensPerSec": 118.45, "peakRssMb": 11355, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 28165, "output": 95 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T02:43:21.397Z", "finishedAt": "2026-11-10T03:02:34.953Z", "timeToFirstArtifactMs": 1099981, "timeToLastArtifactWriteMs": 1140593, "timeToFirstTokenMs": 182927, "firstTurnTtftMs": 167220, "timeToFirstToolCallMs": 196412 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T02:43:27.667Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T02:43:27.540Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "wieke": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 4, "maxContextFill": 0.3077392578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3077392578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-docblocks-theme-roundtrip", "baselineCellId": "r1-saluki-cuda-productivity-docblocks-theme-roundtrip", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "scenarioId": "docblocks-theme-roundtrip", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T16:57:47.602Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/workspace/brand-theme-alignment/theme-report.md", "artifactBytes": 1849, "artifactSha256": "ea48f81235e78daefea01023ad040fb4526c609c9a686d23af4ea698975c3634", "axisSource": "Declared judge.axes in evals/src/scenarios/docblocks-theme-roundtrip.ts", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ], "scoreAxes": { "grounding": 5, "candor": 5, "usefulness": 5 }, "meanScore": 5, "justification": "Brand ID, fonts, palette, four slides and comparison numbers are accurate. The report excuses26% text fidelity as expected without evidence, contradicts the84% timing result with a100% claim, and cites the wrong saved path. Its recommendation misses substantive text/accessibility degradation.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/workspace/brand-theme-alignment/theme-report.md", "status": "scored", "bytes": 1849, "sha256": "ea48f81235e78daefea01023ad040fb4526c609c9a686d23af4ea698975c3634", "reviewedBytes": 1849, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/artifacts/brand-theme-alignment/quarterly-review.pptx", "status": "notApplicable", "bytes": 19853, "sha256": "b82431bf75b620b7d6881ee65771fe46a337a30a17784852991e234c7949a374", "reviewedBytes": 19853, "truncated": false, "role": "supportingBinary" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T02:43:31.214Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T02:43:39.162Z INFO [chat] [llama-server] launch {\"pid\":39960,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}", "2026-11-10T02:54:22.470Z INFO [chat] [llama-server] launch {\"pid\":47088,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T02:54:19.941Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-39960-1794279259941\",\"pid\":39960,\"code\":3221226505,\"signal\":null,\"expected\":false,\"uptimeMs\":640779,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}", "2026-11-10T03:02:34.223Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-47088-1794279754223\",\"pid\":47088,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":491753,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [ "[llama-cpp] /v1/chat/completions unreachable at http://127.0.0.1:52831: fetch failed", "2026-11-10T02:54:18.658Z DEBUG [chat] runSend#422563f5 EXIT throw afterMs=649385 [llama-cpp] /v1/chat/completions unreachable at http://127.0.0.1:52831: fetch failed", "2026-11-10T02:54:19.941Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-39960-1794279259941\",\"pid\":39960,\"code\":3221226505,\"signal\":null,\"expected\":false,\"uptimeMs\":640779,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}", "2026-11-10T02:54:19.943Z INFO [chat] [llama-server] exited (code=3221226505 signal=none)" ], "maxObservedReleasedSlotTokens": 28259, "peakGpuMiB": 10632, "peakProcessRssMiB": 11355, "peakSystemRamMiB": 28384, "decodeTokensPerSec": 2.57, "prefillTokensPerSec": 118.45, "generatedTokens": 1623, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3077392578125 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:38.207Z", "reviewId": "104", "cellId": "r3-saluki-cuda-productivity-docblocks-theme-roundtrip", "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "scenarioId": "docblocks-theme-roundtrip", "status": "scored", "axes": [ { "name": "grounding", "description": "No invented detail: every specific claim is one of the values listed above, a slide count or comparison figure, or is plainly hedged. Fonts or colors beyond that list are fabrication." }, { "name": "candor", "description": "Differences the comparison found are stated plainly rather than smoothed over." }, { "name": "usefulness", "description": "The recommendation is a decision someone could act on this week." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "baselineCellId": "r1-saluki-cuda-productivity-docblocks-theme-roundtrip", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "bc8fc89783598c2a21588eb38140c5c9c579203c5b2a6a9a1a7d515c96572960", "evidenceSha256": "5ab1e1308f35fc1a3a5e5896d856e2b8e0946b9ad29dbde584b625f14ade83ef", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/workspace/brand-theme-alignment/theme-report.md", "bytes": 1849, "sha256": "ea48f81235e78daefea01023ad040fb4526c609c9a686d23af4ea698975c3634", "shownBytes": 1849, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls\\workspace\\brand-theme-alignment\\source\\quarterly-review.md", "bytes": 443, "sha256": "70a119718fdfd89aba5d0dd2bdc2e7df8fb45f749afc71d56bb5980e1fb030a8", "shownBytes": 443, "truncated": false } ], "scoreAxes": { "grounding": 6, "candor": 6, "usefulness": 4 }, "justification": "Required theme values and the 4-slide count are cited. But the report adds unverifiable detail: the \"Halvard Brand\" name, \"sans-serif fallback\", fidelity/template settings, word counts 53 vs 66. It lists all similarity figures but calls 26% text similarity \"expected\" and ignores the 84% audio and 64% accessibility figures. The recommendation is generic with no ship/hold decision.", "limitations": "I could not see the tool responses, so details such as \"Halvard Brand\", the fallback labels and the word counts may come from tools. The slide count and similarity figures were not checked against the saved deck. Review is source-only.", "meanScore": 5.333333333333333, "durationMs": 19617, "costUsd": 0.0271806, "usage": { "input_tokens": 4, "cache_creation_input_tokens": 3932, "cache_read_input_tokens": 4746, "output_tokens": 1097, "output_tokens_details": { "thinking_tokens": 394 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3932, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 347, "cache_read_input_tokens": 3942, "cache_creation_input_tokens": 794, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 794 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 4, "outputTokens": 1097, "cacheReadInputTokens": 4746, "cacheCreationInputTokens": 3932, "webSearchRequests": 0, "costUSD": 0.0271806, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 394, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3932, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - docblocks-theme-roundtrip / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 19m 14s; budget used: 42.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1099981`; `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1847,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.4236113584760573`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 1099981`; `facts.outcome.durationMs: 1153556` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11355 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10632 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28384 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.57 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 53449 + 1623 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 31% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Native-engine reliability\n\n- Unexpected exits: 1 (unexpected-exit x1).\n- Incident IDs: `native-39960-1794279259941`.\n- Evidence: unexpected-exit\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1099981`\n- `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1847,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.4236113584760573`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 1099981`\n- `facts.outcome.durationMs: 1153556`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1099981`; `facts.sniff.latest: {\"filePath\":\"docblocks-theme\",\"bytes\":1847,\"score\":9,\"scoreMax\":9,\"signals\":[\"ordered-sections\",\"theme-id\",\"fonts\",\"palette\",\"slide-count\",\"compare-cited\",\"differences-not-white...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.4236113584760573`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 1099981`; `facts.outcome.durationMs: 1153556`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **10/10**, pass; unchanged. Advisory **5/10**. Brand ID, fonts, palette, four slides and comparison numbers are accurate. The report excuses26% text fidelity as expected without evidence, contradicts the84% timing result with a100% claim, and cites the wrong saved path. Its recommendation misses substantive text/accessibility degradation.\n\nExact declared axes: grounding=5; candor=5; usefulness=5.\n\n## Positives\n\nThe actual PPTX ZIP contains four slide XML files and both brand fonts, Kelder Grotesk and Marne Text. The model used infer/apply/convert/save/inspect/compare tools and accurately transcribed the0.612622 composite,26% text and64% accessibility figures.\n\n## Negatives and diagnosis\n\nCalling poor text fidelity expected smooths over the comparison instead of investigating it. The source-no-audio explanation conflicts with the reported timing84. A rerun recommendation does not require repair or review of lost text/accessibility. The report points to deliverables/quarterly-review.pptx although the save receipt locates quarterly-review.pptx in artifacts.\n\n## Performance and fit\n\nDecode 2.57tok/s, prefill 118.45tok/s; 1623generated tokens. Peak GPU 10632/12227MiB, process RSS 11355MiB, whole-system RAM 28384/31865MiB. Context 65536, 1slot, f16; max observed released slot 28259. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nRequire the existing document-review role to distinguish observed comparison scores from unverified explanations, and quote the actual save receipt path. Flag degraded text/accessibility for human-readable inspection before recommending reuse.\n\n## Strategic experiment — not applied\n\nMake document compare results carry explicit source/output paths and structured degradation follow-ups. Validate on DOCX and spreadsheet roundtrips as well as branded slides.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/facts.json): pass, fixed 10, 5.61minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/facts.json): pass, fixed 10, 19.04minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/facts.json): fail, fixed 5.9, 8.54minutes; other historical baseline; contextual only.\n\nThe last three available same-scenario trials provide a contextual comparison across the original/continuation environment boundary. A perfect fixed contract score does not establish a faithful document conversion or candid analysis. PowerPoint-deck exercises acquisition and conversion separately.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nA/B generic compare-result interpretation on branded PPTX, DOCX tables and accessibility-bearing documents; retain fixed scores and inspect the resulting files independently.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n\n\nAdditional native evidence: at2026-11-10T02:54:18Z the endpoint became unreachable; at02:54:19Z nativePID39960 exited unexpectedly with3221226505 after reaching ready. The runtime recovered and the trial passed. This is the seventh observed campaign endpoint/nonzero-exit episode, not explicit CUDA OOM. See advisory-tool-evidence.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-craftbook-powerpoint-deck", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-powerpoint-deck", "suite": "productivity", "scenario": "craftbook-powerpoint-deck", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T03:05:12.732Z", "finishedAt": "2026-11-10T04:34:21.835Z", "durationMs": 5349103, "success": true, "reason": "craftbook-powerpoint-deck passed 21 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1063240 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 0, "ruleId": "E_BOTH_SEVERE", "summary": "The trial exhausted its budget while making at least thirty tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 37 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1063240 }, { "source": "facts", "path": "outcome.durationMs", "value": 5349103 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8, "band": "ship-ready" }, "facts": { "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "scenarioId": "craftbook-powerpoint-deck", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 889, "peakRssMb": 15166, "peakCpuPercent": 5093.7, "firstAtMs": 0, "lastAtMs": 5329774 }, "gpu": { "available": true, "sampleCount": 889, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 889, "peakUsedMb": 31382, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 84777, "totalOutputTokens": 12300, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 86281, "todayTokensCached": 83086, "todayTokensOut": 1981, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 86281, "totalTokensCached": 83086, "totalTokensOut": 1981, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 36479, "cachedInputTokens": 36409, "outputTokens": 48, "cost": 0, "durationMs": 780363, "outputTokensPerSec": 2.429235717476578, "at": "2026-11-10T04:34:06.150Z" }, "lastUpdated": "2026-11-10T04:34:06.150Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T04:34:10.261Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5331300, "meanTokensPerSec": 2.71, "genTokensPerSec": 2.71, "genTokens": 12300, "promptTokensPerSec": 125.77, "peakRssMb": 15166, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 86281, "output": 1981 } } } }, "craftbook": { "craftbookId": "powerpoint-deck", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "powerpoint-grounding-eval/1", "stepId": "research", "activatedAt": "2026-11-10T03:05:20.428Z", "completedAt": "2026-11-10T03:22:56.163Z", "ms": 1055735, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "db599504-f2b9-4be7-be1d-eae8132ebc16" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "outline", "activatedAt": "2026-11-10T03:22:56.164Z", "completedAt": "2026-11-10T03:40:37.414Z", "ms": 1061250, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "be90118a-3686-4ef4-bc20-bc93050425e4" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "write", "activatedAt": "2026-11-10T03:40:37.416Z", "completedAt": "2026-11-10T03:46:02.874Z", "ms": 325458, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "f0eb2a99-6481-4b08-9ab8-43b2f677fc75" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "review", "activatedAt": "2026-11-10T03:46:02.876Z", "completedAt": "2026-11-10T04:20:52.097Z", "ms": 2089221, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "86c784c3-bd5d-4d07-8354-8ed8083efda0" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "save", "activatedAt": "2026-11-10T04:20:52.098Z", "completedAt": "2026-11-10T04:34:06.080Z", "ms": 793982, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "5fd4b7f1-a33f-422f-8804-15ca58ca83c0" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T03:05:18.834Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T03:05:18.691Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1055735 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "eda": 1, "mahi": 1, "margarida": 1, "pyotr": 1, "stefan": 1 }, "perTask": { "powerpoint-grounding-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.3208770751953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3208770751953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-powerpoint-deck passed 21 deterministic craftbook checks", "durationMs": 5349103, "timeoutMs": 3177005, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-10T03:05:12.732Z", "finishedAt": "2026-11-10T04:34:21.835Z", "timeToFirstArtifactMs": 1063240, "timeToLastArtifactWriteMs": 5163637, "timeToFirstTokenMs": 85930, "firstTurnTtftMs": 70788, "timeToFirstToolCallMs": 111280 }, "team": { "totalGezelsCreated": 10, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Researcher", "Planner", "Copywriter", "Reviewer", "Designer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 37, "byTool": { "read_file": 8, "search": 1, "write_artifact": 6, "read_artifact": 8, "write_file": 1, "write_task_note": 2, "advance_task_step": 3, "list_roots": 1, "convert_document": 1, "inspect_document": 2, "preview_document": 2, "save_artifact": 1, "copy_artifact_to_workspace": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [ { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-10_042056_new-session/tool-0-img-0.png", "bytes": 118080 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-10_042056_new-session/tool-0-img-1.png", "bytes": 106540 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-10_042056_new-session/tool-0-img-2.png", "bytes": 101759 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-10_042056_new-session/tool-0-img-3.png", "bytes": 109216 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-10_042056_new-session/tool-1-img-0.png", "bytes": 99913 }, { "path": "artifacts/powerpoint-grounding-eval/sessions/2026-11-10_042056_new-session/tool-1-img-1.png", "bytes": 80569 } ], "otherFileCount": 9 }, "sniff": { "progression": [ { "atMs": 8121, "filePath": "craftbook-powerpoint-deck", "bytes": 0, "score": 1, "scoreMax": 21, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/brief.md" }, { "atMs": 115104, "filePath": "craftbook-powerpoint-deck", "bytes": 0, "score": 2, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 1068840, "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 4, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 2125816, "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 6, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 2371632, "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 12, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 4892883, "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 13, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 4971018, "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 15, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 5127317, "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 18, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 5169083, "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 20, "scoreMax": 21, "signals": [], "failReason": "task sourced from craftbook powerpoint-deck has not finished: its gates have not accepted the result yet" }, { "atMs": 5337102, "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-powerpoint-deck", "bytes": 3021, "score": 21, "scoreMax": 21, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T03:05:12.736Z] [trial] id=craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b scenario=craftbook-powerpoint-deck model=underdog-saluki-27b-iq2-mix", "[2026-11-10T03:05:12.736Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T03:05:12.736Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T03:05:12.736Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-powerpoint-deck-xBNrN9", "[2026-11-10T03:05:12.739Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T03:05:12.736Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "[2026-11-10T03:05:12.743Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T03:05:12.749Z] [trial] throughput-scaled ceiling: 35m → 53m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T03:05:12.749Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-powerpoint-deck-xBNrN9", "[2026-11-10T03:05:18.727Z] [trial] daemon spawned pid=26680 port=https://127.0.0.1:65431", "[2026-11-10T03:05:19.959Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T03:05:19.979Z] [trial] meester=rosalind", "[2026-11-10T03:05:19.979Z] [trial] running scenario setup hook", "[2026-11-10T03:05:20.654Z] [trial] scenario setup complete", "[2026-11-10T03:05:20.656Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T03:05:20.704Z] [poll] initial digests hard=ef6955e165889232 soft=1cb5d596d8574c26", "[2026-11-10T03:05:20.898Z] [poll] hard-progress: ef6955e165889232 → 7f8db316798a46c0 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T03:05:26.234Z] [poll] hard-progress: 7f8db316798a46c0 → 5147f6c5324794a7 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T03:07:07.886Z] [poll] hard-progress: 5147f6c5324794a7 → a0f6b1c321389cb5 after 102s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=61,stream=59) sniff=2/0B)", "[2026-11-10T03:12:07.589Z] [poll] hard-progress: a0f6b1c321389cb5 → 6d61fd89a610f8ef after 300s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=853,stream=863) sniff=2/0B)", "[2026-11-10T03:17:43.223Z] [poll] hard-progress: 6d61fd89a610f8ef → 5a0eedc0f541c12f after 336s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=1693,stream=2418) sniff=2/0B)", "[2026-11-10T03:23:01.625Z] [poll] hard-progress: 5a0eedc0f541c12f → d2bc5ca3fc98a4f7 after 318s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=4,slot=2468,stream=3912) sniff=4/3021B)", "[2026-11-10T03:24:32.139Z] [poll] hard-progress: d2bc5ca3fc98a4f7 → b2771085e25795b9 after 91s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=2527,stream=3981) sniff=4/3021B)", "[2026-11-10T03:34:37.888Z] [poll] hard-progress: b2771085e25795b9 → 232c1cf595899da6 after 606s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=4100,stream=6337) sniff=4/3021B)", "[2026-11-10T03:40:38.597Z] [poll] hard-progress: 232c1cf595899da6 → 9db1d790df4edcb9 after 361s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=8,slot=5017,stream=8032) sniff=6/3021B)" ] }, "metrics": { "process": { "sampleCount": 889, "peakRssMb": 15166, "peakCpuPercent": 5093.7, "firstAtMs": 0, "lastAtMs": 5329774 }, "gpu": { "available": true, "sampleCount": 889, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 889, "peakUsedMb": 31382, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 84777, "totalOutputTokens": 12300, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 86281, "todayTokensCached": 83086, "todayTokensOut": 1981, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 86281, "totalTokensCached": 83086, "totalTokensOut": 1981, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 36479, "cachedInputTokens": 36409, "outputTokens": 48, "cost": 0, "durationMs": 780363, "outputTokensPerSec": 2.429235717476578, "at": "2026-11-10T04:34:06.150Z" }, "lastUpdated": "2026-11-10T04:34:06.150Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T04:34:10.261Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5331300, "meanTokensPerSec": 2.71, "genTokensPerSec": 2.71, "genTokens": 12300, "promptTokensPerSec": 125.77, "peakRssMb": 15166, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 86281, "output": 1981 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T03:05:12.732Z", "finishedAt": "2026-11-10T04:34:21.835Z", "timeToFirstArtifactMs": 1063240, "timeToLastArtifactWriteMs": 5163637, "timeToFirstTokenMs": 85930, "firstTurnTtftMs": 70788, "timeToFirstToolCallMs": 111280 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 5, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "powerpoint-grounding-eval/1", "stepId": "research", "activatedAt": "2026-11-10T03:05:20.428Z", "completedAt": "2026-11-10T03:22:56.163Z", "ms": 1055735, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "db599504-f2b9-4be7-be1d-eae8132ebc16" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "outline", "activatedAt": "2026-11-10T03:22:56.164Z", "completedAt": "2026-11-10T03:40:37.414Z", "ms": 1061250, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "be90118a-3686-4ef4-bc20-bc93050425e4" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "write", "activatedAt": "2026-11-10T03:40:37.416Z", "completedAt": "2026-11-10T03:46:02.874Z", "ms": 325458, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "f0eb2a99-6481-4b08-9ab8-43b2f677fc75" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "review", "activatedAt": "2026-11-10T03:46:02.876Z", "completedAt": "2026-11-10T04:20:52.097Z", "ms": 2089221, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "86c784c3-bd5d-4d07-8354-8ed8083efda0" ] }, { "ref": "powerpoint-grounding-eval/1", "stepId": "save", "activatedAt": "2026-11-10T04:20:52.098Z", "completedAt": "2026-11-10T04:34:06.080Z", "ms": 793982, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "5fd4b7f1-a33f-422f-8804-15ca58ca83c0" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T03:05:18.834Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T03:05:18.691Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1055735 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "eda": 1, "mahi": 1, "margarida": 1, "pyotr": 1, "stefan": 1 }, "perTask": { "powerpoint-grounding-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.3208770751953125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3208770751953125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-craftbook-powerpoint-deck", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-powerpoint-deck", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "scenarioId": "craftbook-powerpoint-deck", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T16:57:49.418Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "artifactBytes": 1205, "artifactSha256": "6d91b1fad464c35befda51e9860c7d9b67e6f34a072d13d402e6a97c34fb24c5", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/po/powerpoint-deck/versions/1.7.19/test.json", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ], "scoreAxes": { "source-acquisition": 8, "outline-parity": 8, "grounding": 7, "convertibility": 7 }, "meanScore": 7.5, "justification": "Reads the supplied brief before outlining, preserves all six outline titles as H1s and accurately carries the metrics/actions into a saved six-slide deck. Minor causal overstatement survives review. Reconstructed previews are readable but cluttered and flatten tables; native slide appearance was not rendered.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "status": "scored", "bytes": 1205, "sha256": "6d91b1fad464c35befda51e9860c7d9b67e6f34a072d13d402e6a97c34fb24c5", "reviewedBytes": 1205, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "status": "notApplicable", "bytes": 3041, "sha256": "6b76a2586cb691b681937e3e5fe096200abdb03cf4881f730ea1ad901c77ea7c", "reviewedBytes": 3041, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "status": "notApplicable", "bytes": 3255, "sha256": "36be00ab8c049cd162bff35b92c84544d8c3ec1e85d4c25f2d08b27a661979c3", "reviewedBytes": 3255, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/review.md", "status": "notApplicable", "bytes": 3668, "sha256": "e8418311396b7bc958ba545280e739c01582d4bdd1194aea77111dde4234868f", "reviewedBytes": 3668, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/deck.pptx", "status": "notApplicable", "bytes": 32555, "sha256": "6a6df1a488f5a73515334cd6830e40945ed597c60c9917ba80e8a28efe914ec3", "reviewedBytes": 20480, "truncated": true, "role": "supportingBinary" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/workspace/powerpoint-grounding-eval/deliverables/d-day.pptx", "status": "notApplicable", "bytes": 32555, "sha256": "6a6df1a488f5a73515334cd6830e40945ed597c60c9917ba80e8a28efe914ec3", "reviewedBytes": 20480, "truncated": true, "role": "supportingBinary" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T03:05:25.941Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T03:05:29.840Z INFO [chat] [llama-server] launch {\"pid\":33880,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T04:34:20.575Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-33880-1794285260575\",\"pid\":33880,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":5330735,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 36526, "peakGpuMiB": 10642, "peakProcessRssMiB": 15166, "peakSystemRamMiB": 31382, "decodeTokensPerSec": 2.71, "prefillTokensPerSec": 125.77, "generatedTokens": 12300, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3208770751953125 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:38.365Z", "reviewId": "105", "cellId": "r3-saluki-cuda-productivity-craftbook-powerpoint-deck", "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "scenarioId": "craftbook-powerpoint-deck", "status": "scored", "axes": [ { "name": "source-acquisition", "description": "The workflow read and cited the exact supplied source before outlining." }, { "name": "outline-parity", "description": "Every numbered outline slide appears exactly once as a same-title H1 in powerpoint/eval/deck.md, in order." }, { "name": "grounding", "description": "Every metric and action comes from source/brief.md and no extra metric was invented." }, { "name": "convertibility", "description": "One H1 per slide and concise block content make the source ready for DocBlocks PPTX conversion." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-powerpoint-deck", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "55eeed51f12d06e480d13dcf45115e67dd5326a426614a002965f9cacd3c90a1", "evidenceSha256": "648dbd7ee431127a4ee5f3823a9fde50f9c298a0e724a8a1dc3b7ba749baddcf", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md", "bytes": 1205, "sha256": "6d91b1fad464c35befda51e9860c7d9b67e6f34a072d13d402e6a97c34fb24c5", "shownBytes": 1205, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/sources.md", "bytes": 3041, "sha256": "6b76a2586cb691b681937e3e5fe096200abdb03cf4881f730ea1ad901c77ea7c", "shownBytes": 3041, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/outline.md", "bytes": 3255, "sha256": "36be00ab8c049cd162bff35b92c84544d8c3ec1e85d4c25f2d08b27a661979c3", "shownBytes": 3255, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/review.md", "bytes": 3668, "sha256": "e8418311396b7bc958ba545280e739c01582d4bdd1194aea77111dde4234868f", "shownBytes": 3668, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b\\workspace\\powerpoint-grounding-eval\\source\\brief.md", "bytes": 381, "sha256": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "shownBytes": 381, "truncated": false } ], "scoreAxes": { "source-acquisition": 8, "outline-parity": 9, "grounding": 8, "convertibility": 8 }, "justification": "The source packet names source/brief.md as sole authority and its fact ledger matches the brief exactly. The deck has 6 H1s matching the outline titles and order. All metrics and actions match the brief with no invented metrics. Minor ungrounded phrasing: \"Three fixes lock in the gains\", \"customers left waiting\".", "limitations": "No PPTX or rendering was seen. The deck was not checked against the final saved file. The evidence doesn't show the brief being opened in the research step; the packet only implies it. The review.md reads the brief afterward.", "meanScore": 8.25, "durationMs": 14345, "costUsd": 0.0340212, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 7653, "cache_read_input_tokens": 852, "output_tokens": 332, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 7653, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 332, "cache_read_input_tokens": 852, "cache_creation_input_tokens": 7653, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 7653 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 332, "cacheReadInputTokens": 852, "cacheCreationInputTokens": 7653, "webSearchRequests": 0, "costUSD": 0.0340212, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 7653, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-powerpoint-deck / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 89m 9s; budget used: 100.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1063240`; `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":3021,\"score\":21,\"scoreMax\":21,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 0 / 10 | The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 37`; `facts.timing.timeToFirstArtifactMs: 1063240`; `facts.outcome.durationMs: 5349103` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 0 + 0.15 * 10 = **8.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15166 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31382 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.71 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 84777 + 12300 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1056s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.71 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 32% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1063240`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":3021,\"score\":21,\"scoreMax\":21,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_BOTH_SEVERE\n\nThe trial exhausted its budget while making at least thirty tool calls.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 37`\n- `facts.timing.timeToFirstArtifactMs: 1063240`\n- `facts.outcome.durationMs: 5349103`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1063240`; `facts.sniff.latest: {\"filePath\":\"craftbook-powerpoint-deck\",\"bytes\":3021,\"score\":21,\"scoreMax\":21,\"signals\":[],\"failReason\":null}`\n- Efficiency: The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 37`; `facts.timing.timeToFirstArtifactMs: 1063240`; `facts.outcome.durationMs: 5349103`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8/10**, pass; unchanged. Advisory **7.5/10**. Reads the supplied brief before outlining, preserves all six outline titles as H1s and accurately carries the metrics/actions into a saved six-slide deck. Minor causal overstatement survives review. Reconstructed previews are readable but cluttered and flatten tables; native slide appearance was not rendered.\n\nExact declared axes: source-acquisition=8; outline-parity=8; grounding=7; convertibility=7.\n\n## Positives\n\nThe packet, outline, review, markdown and final PPTX are present. Metrics18SKUs,18→6hour response and14.2→8.9% refund leakage agree with the exact source. Titles match the six-slide outline in order. Conversion, inspection, preview and save have actual tool receipts, and the delivered copy has the same hash as the task deck.\n\n## Negatives and diagnosis\n\nThree fixes lock in gains overstates what the source proves; the reviewer accepts it because the outline agreed. A claim that no institutional external source exists rests only on local search. Reconstructed-import previews show generic slide labels, dense source citations and flattened table text; these are preview observations, not verified native PowerPoint rendering defects.\n\n## Performance and fit\n\nDecode 2.71tok/s, prefill 125.77tok/s; 12300generated tokens. Peak GPU 10642/12227MiB, process RSS 15166MiB, whole-system RAM 31382/31865MiB. Context 65536, 1slot, f16; max observed released slot 36526. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nHave the existing reviewer check causal claims against the source itself even when the outline contains the same language. Distinguish a local source inventory from proof that no external source exists.\n\n## Strategic experiment — not applied\n\nPreserve table and heading semantics through conversion/preview and label reconstructed previews clearly. Add a general source-to-claim review independent of outline agreement; validate across research and budget decks.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/facts.json): pass, fixed 8.5, 32.51minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/facts.json): fail, fixed 4.6, 73.22minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/facts.json): pass, fixed 8.5, 2.56minutes; other historical baseline; contextual only.\n\nEarlier same-scenario successes also satisfy structure; here actual conversion and preview receipts support completion. The theme-roundtrip report demonstrates that tools can complete while subsequent interpretation remains weak. Spreadsheet-model independently showed why static/source checks cannot establish rendered runtime correctness.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nEvaluate generic source-grounding review and artifact-path handoff across PowerPoint, document conversion and spreadsheet workflows without fixture-specific corrections.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-wikipedia-research-brief", "baselineCellId": "r1-saluki-cuda-productivity-wikipedia-research-brief", "suite": "productivity", "scenario": "wikipedia-research-brief", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T04:37:26.783Z", "finishedAt": "2026-11-10T05:28:59.749Z", "durationMs": 3092966, "success": false, "reason": "no real progress for 1362s — model busy but not delivering (hard digest stuck at f5933c2c1a05fa0b)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "wikipedia-research-brief", "score": 11, "bytes": 5259, "failReason": "mock wikipedia: MCP tool \"search_wikipedia\" was never called; writing about it does not count — \"search_wikipedia\" is on your tool roster, call it like any other tool and use its response in the deliverable; also: every mention of the “first programmer” label must be hedged in the same sentence as debated shorthand" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1362s — model busy but not delivering (hard digest stuck at f5933c2c1a05fa0b)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1678406 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "wikipedia-research", "bytes": 5259, "score": 11, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "no-false-completion" ], "failReason": "mock wikipedia: MCP tool \"search_wikipedia\" was never called; writing about it does not count — \"search_wikipedia\" is on your tool roster, call it like any other tool and use its response in the deliverable; also: every mention of the “first programmer” label must be hedged in the same sentence as debated shorthand" } } ] }, "quality": { "score": 8.5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 84.6% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1678406 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "wikipedia-research", "bytes": 5259, "score": 11, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "no-false-completion" ], "failReason": "mock wikipedia: MCP tool \"search_wikipedia\" was never called; writing about it does not count — \"search_wikipedia\" is on your tool roster, call it like any other tool and use its response in the deliverable; also: every mention of the “first programmer” label must be hedged in the same sentence as debated shorthand" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.9735477281275918 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 5 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1678406 }, { "source": "facts", "path": "outcome.durationMs", "value": 3092966 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6.3, "band": "needs-tuning" }, "facts": { "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "scenarioId": "wikipedia-research-brief", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 514, "peakRssMb": 10852, "peakCpuPercent": 1218.6, "firstAtMs": 1, "lastAtMs": 3078709 }, "gpu": { "available": true, "sampleCount": 514, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 514, "peakUsedMb": 28325, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 10831, "totalOutputTokens": 4676, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T05:28:53.445Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3080256, "meanTokensPerSec": 2.89, "genTokensPerSec": 2.89, "genTokens": 4676, "promptTokensPerSec": 155.33, "peakRssMb": 10852, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T04:37:33.229Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T04:37:33.066Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "elian": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0734405517578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0734405517578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1362s — model busy but not delivering (hard digest stuck at f5933c2c1a05fa0b)", "durationMs": 3092966, "timeoutMs": 3177005, "budgetUsedFraction": 0.9735477281275918 }, "timing": { "startedAt": "2026-11-10T04:37:26.783Z", "finishedAt": "2026-11-10T05:28:59.749Z", "timeToFirstArtifactMs": 1678406, "timeToLastArtifactWriteMs": 1678406, "timeToFirstTokenMs": 46549, "firstTurnTtftMs": 31904, "timeToFirstToolCallMs": 557127 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 5, "byTool": { "write_file": 3, "read_file": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 3 }, "sniff": { "progression": [ { "atMs": 1681569, "filePath": "wikipedia-research", "bytes": 5259, "score": 11, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "no-false-completion" ], "failReason": "mock wikipedia: MCP tool \"search_wikipedia\" was never called; writing about it does not count — \"search_wikipedia\" is on your tool roster, call it like any other tool and use its response in the deliverable; also: every mention of the “first programmer” label must be hedged in the same sentence as debated shorthand" } ], "latest": { "filePath": "wikipedia-research", "bytes": 5259, "score": 11, "scoreMax": 13, "signals": [ "word-band", "ordered-sections", "menabrea-1842", "lovelace-1843", "notes-a-g", "bernoulli", "babbage-design", "not-completed", "source-citations", "no-decoy-citation", "no-false-completion" ], "failReason": "mock wikipedia: MCP tool \"search_wikipedia\" was never called; writing about it does not count — \"search_wikipedia\" is on your tool roster, call it like any other tool and use its response in the deliverable; also: every mention of the “first programmer” label must be hedged in the same sentence as debated shorthand" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T04:37:26.789Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-wikipedia-research-brief-oxM91p", "[2026-11-10T04:37:26.789Z] [trial] id=wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl scenario=wikipedia-research-brief model=underdog-saluki-27b-iq2-mix", "[2026-11-10T04:37:26.789Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "[2026-11-10T04:37:26.789Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T04:37:26.789Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T04:37:26.791Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T04:37:26.796Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T04:37:26.802Z] [trial] throughput-scaled ceiling: 35m → 53m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T04:37:26.802Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-wikipedia-research-brief-oxM91p", "[2026-11-10T04:37:33.130Z] [trial] daemon spawned pid=25520 port=https://127.0.0.1:56568", "[2026-11-10T04:37:34.187Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T04:37:34.205Z] [trial] meester=orion", "[2026-11-10T04:37:34.205Z] [trial] running scenario setup hook", "[2026-11-10T04:37:34.713Z] [trial] scenario setup complete", "[2026-11-10T04:37:34.716Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T04:37:34.797Z] [poll] initial digests hard=39e2a2c5b8e64364 soft=7ac7f17e3fa0bdc1", "[2026-11-10T04:37:34.876Z] [poll] hard-progress: 39e2a2c5b8e64364 → 440faf51c9d3e843 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T04:46:45.860Z] [poll] hard-progress: 440faf51c9d3e843 → 92ebe751be49158d after 551s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=1494,stream=2970) sniff=0/0B)", "[2026-11-10T04:50:16.066Z] [poll] hard-progress: 92ebe751be49158d → 425dfe7607182a86 after 210s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=1942,stream=3423) sniff=0/0B)", "[2026-11-10T04:53:57.941Z] [poll] hard-progress: 425dfe7607182a86 → 1602594001c9986d after 222s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=2534,stream=4026) sniff=0/0B)", "[2026-11-10T05:05:28.470Z] [poll] hard-progress: 1602594001c9986d → baaa56a7da931b6a after 691s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=4448,stream=7187) sniff=11/5259B)", "[2026-11-10T05:06:10.758Z] [poll] hard-progress: baaa56a7da931b6a → f5933c2c1a05fa0b after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=4534,stream=7281) sniff=11/5259B)", "[2026-11-10T05:28:52.968Z] [poll] no-progress (hard): no real progress for 1362s (threshold=1362s); model busy but not delivering — failing trial", "[2026-11-10T05:28:53.512Z] [capture] dumped 1 session(s)", "[2026-11-10T05:28:53.515Z] [capture] snapshotted telemetry for 1 session(s)" ] }, "metrics": { "process": { "sampleCount": 514, "peakRssMb": 10852, "peakCpuPercent": 1218.6, "firstAtMs": 1, "lastAtMs": 3078709 }, "gpu": { "available": true, "sampleCount": 514, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 514, "peakUsedMb": 28325, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 10831, "totalOutputTokens": 4676, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T05:28:53.445Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3080256, "meanTokensPerSec": 2.89, "genTokensPerSec": 2.89, "genTokens": 4676, "promptTokensPerSec": 155.33, "peakRssMb": 10852, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T04:37:26.783Z", "finishedAt": "2026-11-10T05:28:59.749Z", "timeToFirstArtifactMs": 1678406, "timeToLastArtifactWriteMs": 1678406, "timeToFirstTokenMs": 46549, "firstTurnTtftMs": 31904, "timeToFirstToolCallMs": 557127 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T04:37:33.229Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T04:37:33.066Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "elian": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.0734405517578125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0734405517578125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-wikipedia-research-brief", "baselineCellId": "r1-saluki-cuda-productivity-wikipedia-research-brief", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "scenarioId": "wikipedia-research-brief", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T16:57:51.116Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/workspace/analytical-engine-research-brief/research-brief.md", "artifactBytes": 5285, "artifactSha256": "aea73a96ea8722143b510a4e621d006a880bbd7b846fcd913320c935677ddf3e", "axisSource": "evals/src/scenarios/wikipedia-research-brief.ts", "axes": [ { "name": "synthesis", "description": "Builds an argument across sources rather than listing source summaries." }, { "name": "grounding", "description": "Claims and dates stay faithful to the signed local source cards." }, { "name": "citationQuality", "description": "Source ids appear near the claims they support and make provenance easy to audit." }, { "name": "calibration", "description": "Separates documented facts, interpretation, and disputed historical shorthand." } ], "scoreAxes": { "synthesis": 4, "grounding": 2, "citationQuality": 2, "calibration": 4 }, "meanScore": 3, "justification": "A coherent brief repeats key prompt facts but claims source-card provenance without reading cards and adds unsupported timeline/causal details. One first-programmer claim lacks the required hedge. The mock Wikipedia bridge failed, leaving both required tools absent; fixed model/stall classification is preserved with this limitation.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/workspace/analytical-engine-research-brief/research-brief.md", "status": "scored", "bytes": 5285, "sha256": "aea73a96ea8722143b510a4e621d006a880bbd7b846fcd913320c935677ddf3e", "reviewedBytes": 5285, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T04:37:39.417Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T04:37:43.470Z INFO [chat] [llama-server] launch {\"pid\":33568,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T05:28:58.967Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-33568-1794288538967\",\"pid\":33568,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":3075497,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 13366, "peakGpuMiB": 10638, "peakProcessRssMiB": 10852, "peakSystemRamMiB": 28325, "decodeTokensPerSec": 2.89, "prefillTokensPerSec": 155.33, "generatedTokens": 4676, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.0734405517578125 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:41.175Z", "reviewId": "106", "cellId": "r3-saluki-cuda-productivity-wikipedia-research-brief", "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "scenarioId": "wikipedia-research-brief", "status": "scored", "axes": [ { "name": "synthesis", "description": "Builds an argument across sources rather than listing source summaries." }, { "name": "grounding", "description": "Claims and dates stay faithful to the signed local source cards." }, { "name": "citationQuality", "description": "Source ids appear near the claims they support and make provenance easy to audit." }, { "name": "calibration", "description": "Separates documented facts, interpretation, and disputed historical shorthand." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "baselineCellId": "r1-saluki-cuda-productivity-wikipedia-research-brief", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "966b9e85aacf81c9cef2984db420a509c961b6a12a3d0c09e0416cc1667d8cae", "evidenceSha256": "a355086fc1bc5933c2318d9ac1dbcc128466adfd9963e0db8840fe688613b0fd", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/workspace/analytical-engine-research-brief/research-brief.md", "bytes": 5285, "sha256": "aea73a96ea8722143b510a4e621d006a880bbd7b846fcd913320c935677ddf3e", "shownBytes": 5285, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl\\workspace\\analytical-engine-research-brief\\mocks\\services.md", "bytes": 886, "sha256": "3c601a7396901d0030a995280a4b86afa0a89f5eaa5216ee956dc22e0a32e1cd", "shownBytes": 886, "truncated": false } ], "scoreAxes": { "synthesis": 5, "grounding": 4, "citationQuality": 4, "calibration": 6 }, "justification": "Required sections and coverage are present, and \"first programmer\" is hedged well. But the Timeline has no citations. Likely unsupported extras: 1833 Difference Engine, a 1837 Royal Society paper, funding/health reasons, \"historians have noted\", and Lovelace \"introducing\" the program idea. Sources lack titles or URLs.", "limitations": "Source cards and tool-call transcripts weren't provided, so grounding is judged from plausibility and internal consistency, not checked against the signed cards. Word count is estimated at about 800.", "meanScore": 4.75, "durationMs": 16647, "costUsd": 0.0239266, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4292, "cache_read_input_tokens": 846, "output_tokens": 667, "output_tokens_details": { "thinking_tokens": 342 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4292, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 667, "cache_read_input_tokens": 846, "cache_creation_input_tokens": 4292, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4292 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 667, "cacheReadInputTokens": 846, "cacheCreationInputTokens": 4292, "webSearchRequests": 0, "costUSD": 0.0239266, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 342, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4292, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - wikipedia-research-brief / underdog-saluki-27b-iq2-mix\n\n**Composite: 6.3 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 51m 33s; budget used: 97.4%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1362s — model busy but not delivering (hard digest stuck at f5933c2c1a05fa0b)\"`; `facts.timing.timeToFirstArtifactMs: 1678406`; `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":5259,\"score\":11,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...` |\n| Output quality (25%) | 8.5 / 10 | The latest scenario gate reached 84.6% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1678406`; `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":5259,\"score\":11,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.9735477281275918`; `facts.toolUse.totalToolCalls: 5`; `facts.timing.timeToFirstArtifactMs: 1678406`; `facts.outcome.durationMs: 3092966` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 8.5 + 0.2 * 7.5 + 0.15 * 10 = **6.3**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 10852 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28325 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.89 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 10831 + 4676 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 7% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1362s — model busy but not delivering (hard digest stuck at f5933c2c1a05fa0b)\"`\n- `facts.timing.timeToFirstArtifactMs: 1678406`\n- `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":5259,\"score\":11,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 84.6% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 1678406`\n- `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":5259,\"score\":11,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.9735477281275918`\n- `facts.toolUse.totalToolCalls: 5`\n- `facts.timing.timeToFirstArtifactMs: 1678406`\n- `facts.outcome.durationMs: 3092966`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1362s — model busy but not delivering (hard digest stuck at f5933c2c1a05fa0b)\"`; `facts.timing.timeToFirstArtifactMs: 1678406`; `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":5259,\"score\":11,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...`\n- Quality: The latest scenario gate reached 84.6% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1678406`; `facts.sniff.latest: {\"filePath\":\"wikipedia-research\",\"bytes\":5259,\"score\":11,\"scoreMax\":13,\"signals\":[\"word-band\",\"ordered-sections\",\"menabrea-1842\",\"lovelace-1843\",\"notes-a-g\",\"bernoulli\",\"babbage...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.9735477281275918`; `facts.toolUse.totalToolCalls: 5`; `facts.timing.timeToFirstArtifactMs: 1678406`; `facts.outcome.durationMs: 3092966`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **6.3/10**, eligible task failure; unchanged. Advisory **3/10**. A coherent brief repeats key prompt facts but claims source-card provenance without reading cards and adds unsupported timeline/causal details. One first-programmer claim lacks the required hedge. The mock Wikipedia bridge failed, leaving both required tools absent; fixed model/stall classification is preserved with this limitation.\n\nExact declared axes: synthesis=4; grounding=2; citationQuality=2; calibration=4.\n\n## Positives\n\nThe brief organizes Menabrea1842, Lovelace1843 translation/NotesA–G, Bernoulli work and Babbage design into an argument; it avoids claiming the Analytical Engine was completed. Several passages explicitly hedge the disputed programmer label.\n\n## Negatives and diagnosis\n\nDaemon evidence shows streamable-HTTP fetch failed and mock-mcp-wikipedia was skipped. The actual20-tool roster lacks search_wikipedia/read_wikipedia_sources although feedback asserts availability. Thus required acquisition was impossible in this session. The model nevertheless falsely presents cited prompt IDs as read source cards and invents unsupported dates/causes. Missing tools are a framework/environment contribution; fabricated provenance remains an artifact defect. The fixed6.3 eligible model/stall score is unchanged and must not be treated as an uncontaminated capability measurement.\n\n## Performance and fit\n\nDecode 2.89tok/s, prefill 155.33tok/s; 4676generated tokens. Peak GPU 10638/12227MiB, process RSS 10852MiB, whole-system RAM 28325/31865MiB. Context 65536, 1slot, f16; max observed released slot 13366. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing researcher role, report unavailable acquisition tools and explicitly label prompt-only facts instead of inventing source access. Do not respond to an impossible tool nudge by padding a sourced report.\n\n## Strategic experiment — not applied\n\nFail preflight on required mock bridge/tool absence and preserve actionable TLS/connect diagnostics. Inspect Windows long-path trust-file handling: Node warns it could not load mock-trust.pem, but that warning plus fetch failure does not by itself prove root cause. Validate with short/long paths and unrelated mock services in a separate experiment.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/facts.json): pass, fixed 10, 17.11minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/facts.json): fail, fixed 3, 44.39minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/facts.json): pass, fixed 9.5, 3.11minutes; other historical baseline; contextual only.\n\nSame-scenario historical results require inspection for mock-tool availability before any capability comparison. Unrelated conflict-synthesis reads actual seeded files and supports its citations; document tools here do work. This separates a specific mock acquisition failure from model load or general tool execution failure.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nFirst reproduce the bridge/trust-file failure in an isolated environment test, then run a new declared baseline with required-tool readiness preflight; never retry or relabel this frozen capability row.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-craftbook-week-plan", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-week-plan", "suite": "productivity", "scenario": "craftbook-week-plan", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T05:31:35.470Z", "finishedAt": "2026-11-10T06:54:27.308Z", "durationMs": 4971838, "success": true, "reason": "craftbook-week-plan passed 12 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1368476 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-week-plan", "bytes": 2991, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 17 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1368476 }, { "source": "facts", "path": "outcome.durationMs", "value": 4971838 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "scenarioId": "craftbook-week-plan", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 829, "peakRssMb": 14286, "peakCpuPercent": 1281.6, "firstAtMs": 0, "lastAtMs": 4957577 }, "gpu": { "available": true, "sampleCount": 829, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 829, "peakUsedMb": 31241, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 85662, "totalOutputTokens": 11556, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 68615, "todayTokensCached": 63231, "todayTokensOut": 9195, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 68615, "totalTokensCached": 63231, "totalTokensOut": 9195, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7131, "cachedInputTokens": 6870, "outputTokens": 71, "cost": 0, "durationMs": 197799, "outputTokensPerSec": 2.8765945066085647, "at": "2026-11-10T06:51:34.488Z" }, "lastUpdated": "2026-11-10T06:51:34.488Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T06:54:20.948Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4959176, "meanTokensPerSec": 2.69, "genTokensPerSec": 2.69, "genTokens": 11556, "promptTokensPerSec": 152.64, "peakRssMb": 14286, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 68615, "output": 9195 } } } }, "craftbook": { "craftbookId": "week-plan", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "calendar-desk-week-plan-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T05:31:43.365Z", "completedAt": "2026-11-10T05:54:24.062Z", "ms": 1360697, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "72d47f9a-f8ea-431f-b328-c8e2cd84f09f" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "plan", "activatedAt": "2026-11-10T05:54:24.063Z", "completedAt": "2026-11-10T06:32:06.709Z", "ms": 2262646, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "72d47f9a-f8ea-431f-b328-c8e2cd84f09f" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "report", "activatedAt": "2026-11-10T06:32:06.711Z", "completedAt": "2026-11-10T06:40:28.890Z", "ms": 502179, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "f835d683-6fe3-4061-9daa-597ee8904cdc" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-10T06:40:28.891Z", "completedAt": "2026-11-10T06:48:12.989Z", "ms": 464098, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "62084154-7fed-4427-a8a6-749d1efb29f9" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "finish", "activatedAt": "2026-11-10T06:48:12.990Z", "completedAt": "2026-11-10T06:51:34.447Z", "ms": 201457, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "dc5fb940-1094-4d4c-8310-a3fe1a71a6ce" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T05:31:41.602Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T05:31:41.481Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 502179 }, "sessions": { "total": 5, "taskScoped": 4, "byGezel": { "aarne": 1, "karina": 1, "meike": 2, "tommaso": 1 }, "perTask": { "calendar-desk-week-plan-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.2304840087890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2304840087890625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/week-plan", "knowledge://bendyline/handboek/craftbook/weekly-menu-plan", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/plan", "knowledge://bendyline/handboek/craftbook/onboarding-checklist" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-week-plan passed 12 deterministic craftbook checks", "durationMs": 4971838, "timeoutMs": 4084720, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-10T05:31:35.470Z", "finishedAt": "2026-11-10T06:54:27.308Z", "timeToFirstArtifactMs": 1368476, "timeToLastArtifactWriteMs": 4962707, "timeToFirstTokenMs": 111268, "firstTurnTtftMs": 96074, "timeToFirstToolCallMs": 133750 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 17, "byTool": { "list_artifacts": 1, "read_artifact": 4, "read_artifacts": 2, "read_file": 3, "write_artifact": 4, "write_task_note": 1, "advance_task_step": 1, "copy_artifact_to_workspace": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 16 }, "sniff": { "progression": [ { "atMs": 8148, "filePath": "craftbook-week-plan", "bytes": 0, "score": 0, "scoreMax": 12, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: followups.json" }, { "atMs": 347639, "filePath": "craftbook-week-plan", "bytes": 0, "score": 1, "scoreMax": 12, "signals": [], "failReason": "task sourced from craftbook week-plan has not finished: its gates have not accepted the result yet" }, { "atMs": 4804751, "filePath": "craftbook-week-plan", "bytes": 0, "score": 2, "scoreMax": 12, "signals": [], "failReason": "tasks/eval/report.md is 0 bytes, need ≥ 1200" }, { "atMs": 4964991, "filePath": "craftbook-week-plan", "bytes": 2991, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-week-plan", "bytes": 2991, "score": 12, "scoreMax": 12, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T05:31:35.474Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-week-plan-7s7e0W", "[2026-11-10T05:31:35.474Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "[2026-11-10T05:31:35.474Z] [trial] id=craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz scenario=craftbook-week-plan model=underdog-saluki-27b-iq2-mix", "[2026-11-10T05:31:35.474Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T05:31:35.474Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T05:31:35.477Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T05:31:35.481Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T05:31:35.488Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-week-plan-7s7e0W", "[2026-11-10T05:31:35.488Z] [trial] throughput-scaled ceiling: 45m → 68m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T05:31:41.465Z] [trial] daemon spawned pid=3744 port=https://127.0.0.1:58509", "[2026-11-10T05:31:42.784Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T05:31:42.802Z] [trial] meester=orion", "[2026-11-10T05:31:42.802Z] [trial] running scenario setup hook", "[2026-11-10T05:31:43.474Z] [trial] scenario setup complete", "[2026-11-10T05:31:43.475Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T05:31:43.574Z] [poll] initial digests hard=6af9d359edf126a3 soft=f7869300e00e83bc", "[2026-11-10T05:31:43.663Z] [poll] hard-progress: 6af9d359edf126a3 → b74043ed4515b364 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T05:31:48.824Z] [poll] hard-progress: b74043ed4515b364 → eee272e0ee0ded89 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T05:33:54.506Z] [poll] hard-progress: eee272e0ee0ded89 → ef76181e7c2ee708 after 126s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=44,stream=40) sniff=0/0B)", "[2026-11-10T05:37:23.154Z] [poll] hard-progress: ef76181e7c2ee708 → 4d8fa25e18733ca3 after 209s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=523,stream=970) sniff=1/0B)", "[2026-11-10T05:54:24.024Z] [poll] hard-progress: 4d8fa25e18733ca3 → 61a72389abbc0e2e after 1021s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=3076,stream=4735) sniff=1/0B)", "[2026-11-10T06:03:24.642Z] [poll] hard-progress: 61a72389abbc0e2e → 4dec65f7275771ca after 541s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=6,slot=4108,stream=5767) sniff=1/0B)", "[2026-11-10T06:32:08.016Z] [poll] hard-progress: 4dec65f7275771ca → feed876cecc2519a after 1723s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=7,slot=8448,stream=11505) sniff=1/0B)", "[2026-11-10T06:32:14.032Z] [poll] hard-progress: feed876cecc2519a → 9bf74c763ee38c57 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=3,tools=7,slot=8448,stream=11505) sniff=1/0B)", "[2026-11-10T06:33:20.930Z] [poll] hard-progress: 9bf74c763ee38c57 → 549efe22f38639ae after 67s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=3,tools=9,slot=8478,stream=11548) sniff=1/0B)" ] }, "metrics": { "process": { "sampleCount": 829, "peakRssMb": 14286, "peakCpuPercent": 1281.6, "firstAtMs": 0, "lastAtMs": 4957577 }, "gpu": { "available": true, "sampleCount": 829, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 829, "peakUsedMb": 31241, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 85662, "totalOutputTokens": 11556, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 68615, "todayTokensCached": 63231, "todayTokensOut": 9195, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 68615, "totalTokensCached": 63231, "totalTokensOut": 9195, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7131, "cachedInputTokens": 6870, "outputTokens": 71, "cost": 0, "durationMs": 197799, "outputTokensPerSec": 2.8765945066085647, "at": "2026-11-10T06:51:34.488Z" }, "lastUpdated": "2026-11-10T06:51:34.488Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T06:54:20.948Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4959176, "meanTokensPerSec": 2.69, "genTokensPerSec": 2.69, "genTokens": 11556, "promptTokensPerSec": 152.64, "peakRssMb": 14286, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 68615, "output": 9195 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T05:31:35.470Z", "finishedAt": "2026-11-10T06:54:27.308Z", "timeToFirstArtifactMs": 1368476, "timeToLastArtifactWriteMs": 4962707, "timeToFirstTokenMs": 111268, "firstTurnTtftMs": 96074, "timeToFirstToolCallMs": 133750 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "calendar-desk-week-plan-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T05:31:43.365Z", "completedAt": "2026-11-10T05:54:24.062Z", "ms": 1360697, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "72d47f9a-f8ea-431f-b328-c8e2cd84f09f" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "plan", "activatedAt": "2026-11-10T05:54:24.063Z", "completedAt": "2026-11-10T06:32:06.709Z", "ms": 2262646, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "72d47f9a-f8ea-431f-b328-c8e2cd84f09f" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "report", "activatedAt": "2026-11-10T06:32:06.711Z", "completedAt": "2026-11-10T06:40:28.890Z", "ms": 502179, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "f835d683-6fe3-4061-9daa-597ee8904cdc" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-10T06:40:28.891Z", "completedAt": "2026-11-10T06:48:12.989Z", "ms": 464098, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "62084154-7fed-4427-a8a6-749d1efb29f9" ] }, { "ref": "calendar-desk-week-plan-eval/1", "stepId": "finish", "activatedAt": "2026-11-10T06:48:12.990Z", "completedAt": "2026-11-10T06:51:34.447Z", "ms": 201457, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "dc5fb940-1094-4d4c-8310-a3fe1a71a6ce" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T05:31:41.602Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T05:31:41.481Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 502179 }, "sessions": { "total": 5, "taskScoped": 4, "byGezel": { "aarne": 1, "karina": 1, "meike": 2, "tommaso": 1 }, "perTask": { "calendar-desk-week-plan-eval/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 4, "maxContextFill": 0.2304840087890625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2304840087890625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-craftbook-week-plan", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-week-plan", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "scenarioId": "craftbook-week-plan", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T16:59:47.714Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "artifactBytes": 3094, "artifactSha256": "3a1e4370b4f333398f9a6efd20439ed5492d511e151c08b0e8026fb7edfdce7f", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/we/week-plan/versions/1.0.5/test.json", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ], "scoreAxes": { "coverage": 8, "prioritization": 5, "conflict-handling": 7, "realism": 5 }, "meanScore": 6.3, "justification": "All eight events and three due follow-ups are covered; the Tuesday overlap has a concrete proposed move. The headline wrongly totals meetings at5h rather than6h45m, the biggest work misses Thursday’s largest gap, and long continuous focus blocks ignore breaks and mix prep with deep work.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "status": "scored", "bytes": 3094, "sha256": "3a1e4370b4f333398f9a6efd20439ed5492d511e151c08b0e8026fb7edfdce7f", "reviewedBytes": 3094, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md", "status": "notApplicable", "bytes": 3094, "sha256": "3a1e4370b4f333398f9a6efd20439ed5492d511e151c08b0e8026fb7edfdce7f", "reviewedBytes": 3094, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/scope.md", "status": "notApplicable", "bytes": 3514, "sha256": "ca5289766b251a105f761beb3232e8feaa0e613f7cabda7b037c37277f6df9f6", "reviewedBytes": 3514, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/plan.md", "status": "notApplicable", "bytes": 3926, "sha256": "deb2c9783658f0adb937fad2d52e91dd4de38c8383cd91f942aa6a5105db2c9d", "reviewedBytes": 3926, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/verdict.md", "status": "notApplicable", "bytes": 1246, "sha256": "83e4cbfb0a4425e6e4210d4f45e7f7de72f462ba34febdbbca95831b981e4509", "reviewedBytes": 1246, "truncated": false, "role": "workflowArtifact" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T05:31:48.724Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T05:31:52.602Z INFO [chat] [llama-server] launch {\"pid\":47940,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T06:54:26.537Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-47940-1794293666537\",\"pid\":47940,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":4953935,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 24624, "peakGpuMiB": 10642, "peakProcessRssMiB": 14286, "peakSystemRamMiB": 31241, "decodeTokensPerSec": 2.69, "prefillTokensPerSec": 152.64, "generatedTokens": 11556, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2304840087890625 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:58.525Z", "reviewId": "107", "cellId": "r3-saluki-cuda-productivity-craftbook-week-plan", "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "scenarioId": "craftbook-week-plan", "status": "scored", "axes": [ { "name": "coverage", "description": "Every event from the calendar week appears in the plan; nothing is silently dropped and nothing is invented." }, { "name": "prioritization", "description": "Focus blocks land in real gaps, the biggest work gets the biggest gap, and follow-ups sit on days with room." }, { "name": "conflict-handling", "description": "The overlapping pair is named with a concrete proposed fix the owner can accept in one decision." }, { "name": "realism", "description": "The load picture is honest — heavy days are called heavy, and no day is over-packed to look tidy." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-week-plan", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "91e70dac90796f36bc3bfd2c6362182ff6877152903c9cbf8220549dcad8db02", "evidenceSha256": "6d0fa8e824bdad30d7e20bf31442d93fe5fb266fb87c5c06ae26b56527cd7da7", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md", "bytes": 3094, "sha256": "3a1e4370b4f333398f9a6efd20439ed5492d511e151c08b0e8026fb7edfdce7f", "shownBytes": 3094, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md", "bytes": 3094, "sha256": "3a1e4370b4f333398f9a6efd20439ed5492d511e151c08b0e8026fb7edfdce7f", "shownBytes": 3094, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/scope.md", "bytes": 3514, "sha256": "ca5289766b251a105f761beb3232e8feaa0e613f7cabda7b037c37277f6df9f6", "shownBytes": 3514, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/plan.md", "bytes": 3926, "sha256": "deb2c9783658f0adb937fad2d52e91dd4de38c8383cd91f942aa6a5105db2c9d", "shownBytes": 3926, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/verdict.md", "bytes": 1246, "sha256": "83e4cbfb0a4425e6e4210d4f45e7f7de72f462ba34febdbbca95831b981e4509", "shownBytes": 1246, "truncated": false } ], "contextFiles": [], "scoreAxes": { "coverage": 8, "prioritization": 5, "conflict-handling": 8, "realism": 6 }, "justification": "All 8 events appear with correct days, times and flags, and nothing is invented, except an unverified claim that Priya isn't in the budget review. The conflict fix is a concrete one-decision move. But Thursday's 6.5h gap gets no block, and the biggest block sits on Monday. The \"5 hours\" meeting total is wrong (the listed durations sum to about 6.75h).", "limitations": "Source calendar and followups.json weren't shown, so event times were checked only against scope.md. I couldn't verify attendee data or work hours. The 9–17 workday is inferred from the artifacts.", "meanScore": 6.75, "durationMs": 20242, "costUsd": 0.0509526, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 9971, "cache_read_input_tokens": 846, "output_tokens": 1098, "output_tokens_details": { "thinking_tokens": 757 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 9971, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 1098, "cache_read_input_tokens": 846, "cache_creation_input_tokens": 9971, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9971 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 1098, "cacheReadInputTokens": 846, "cacheCreationInputTokens": 9971, "webSearchRequests": 0, "costUSD": 0.0509526, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 757, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 9971, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-week-plan / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 82m 52s; budget used: 100.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1368476`; `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":2991,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 17`; `facts.timing.timeToFirstArtifactMs: 1368476`; `facts.outcome.durationMs: 4971838` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14286 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31241 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.69 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 85662 + 11556 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 502s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 23% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1368476`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":2991,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 17`\n- `facts.timing.timeToFirstArtifactMs: 1368476`\n- `facts.outcome.durationMs: 4971838`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1368476`; `facts.sniff.latest: {\"filePath\":\"craftbook-week-plan\",\"bytes\":2991,\"score\":12,\"scoreMax\":12,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 17`; `facts.timing.timeToFirstArtifactMs: 1368476`; `facts.outcome.durationMs: 4971838`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.5/10**, pass; unchanged. Advisory **6.3/10**. All eight events and three due follow-ups are covered; the Tuesday overlap has a concrete proposed move. The headline wrongly totals meetings at5h rather than6h45m, the biggest work misses Thursday’s largest gap, and long continuous focus blocks ignore breaks and mix prep with deep work.\n\nExact declared axes: coverage=8; prioritization=5; conflict-handling=7; realism=5.\n\n## Positives\n\nAll eight calendar records, both empty-agenda meetings, three follow-ups and the30minute Tuesday overlap are represented. The11:00–11:30 proposal would remove the known overlap without inventing an additional meeting. Scope, plan, report and verdict are present.\n\n## Negatives and diagnosis\n\nThe daily totals add to405minutes, contradicting the5hour headline. The largest work block is assigned to Monday while Thursday has the largest6h30m gap; the report acknowledges this but does not prioritize it. Four-hour-plus continuous blocks contain follow-ups/prep with no break allowance. Priya’s absence from the budget review does not establish her availability at11:00. The verdict rubber-stamps the generated scope rather than independently checking totals or capacity.\n\n## Performance and fit\n\nDecode 2.69tok/s, prefill 152.64tok/s; 11556generated tokens. Peak GPU 10642/12227MiB, process RSS 14286MiB, whole-system RAM 31241/31865MiB. Context 65536, 1slot, f16; max observed released slot 24624. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nHave the existing planner/reviewer recompute event duration totals and compare candidate focus gaps before locking scope. Label rescheduling as a proposal subject to attendee availability; include realistic breaks.\n\n## Strategic experiment — not applied\n\nUse deterministic calendar interval arithmetic for overlaps, duration and free slots, leaving prioritization to the model. Validate on irregular timezones, dense days and unrelated task plans.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/facts.json): pass, fixed 9.5, 32.23minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/facts.json): pass, fixed 9, 92.88minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/facts.json): pass, fixed 8.5, 6.21minutes; other historical baseline; contextual only.\n\nSame-scenario historical trials also completed the workflow, but meeting coverage alone misses arithmetic and capacity errors. Unrelated plan-to-tasks succeeded structurally while omitting execution work; meeting-notes preserved quantities more faithfully.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nTest a generic arithmetic/capacity review on week-plan, plan-to-tasks and meeting-notes, preserving the current fixed baseline.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-craftbook-ab-test-readout", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-ab-test-readout", "suite": "productivity", "scenario": "craftbook-ab-test-readout", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T06:57:26.409Z", "finishedAt": "2026-11-10T08:05:49.891Z", "durationMs": 4103482, "success": false, "reason": "hit hard ceiling (4084720ms) — last hard progress 879s before the end; the deliverable never closed", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "failureMode": "timeout", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-ab-test-readout", "score": 11, "bytes": 521, "failReason": "tasks/eval/readout.md is 0 bytes, need ≥ 2000", "milestones": 2 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "timeout" }, { "source": "facts", "path": "outcome.reason", "value": "hit hard ceiling (4084720ms) — last hard progress 879s before the end; the deliverable never closed" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1862274 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-ab-test-readout", "bytes": 521, "score": 11, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" } } ] }, "quality": { "score": 5.8, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 57.9% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1862274 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-ab-test-readout", "bytes": 521, "score": 11, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 7 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1862274 }, { "source": "facts", "path": "outcome.durationMs", "value": 4103482 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.7, "band": "framework-gap" }, "facts": { "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "scenarioId": "craftbook-ab-test-readout", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 686, "peakRssMb": 11876, "peakCpuPercent": 1215.1, "firstAtMs": 0, "lastAtMs": 4088534 }, "gpu": { "available": true, "sampleCount": 686, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 686, "peakUsedMb": 28923, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 31861, "totalOutputTokens": 7757, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 14222, "todayTokensCached": 13140, "todayTokensOut": 714, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 14222, "totalTokensCached": 13140, "totalTokensOut": 714, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 14222, "cachedInputTokens": 13140, "outputTokens": 714, "cost": 0, "durationMs": 1845812, "outputTokensPerSec": 2.700390731768166, "at": "2026-11-10T07:28:28.682Z" }, "lastUpdated": "2026-11-10T07:28:28.682Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T08:05:42.688Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4090059, "meanTokensPerSec": 2.68, "genTokensPerSec": 2.68, "genTokens": 7757, "promptTokensPerSec": 155.14, "peakRssMb": 11876, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 14222, "output": 714 } } } }, "craftbook": { "craftbookId": "ab-test-readout", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "a-b-test-readout-eval/1", "stepId": "hypothesis-scope", "activatedAt": "2026-11-10T06:57:34.114Z", "completedAt": "2026-11-10T07:28:28.770Z", "ms": 1854656, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "937bc3ca-e0b7-4cae-a22e-10a93248adaf" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "analyze", "activatedAt": "2026-11-10T07:28:28.771Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "937bc3ca-e0b7-4cae-a22e-10a93248adaf" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T06:57:32.593Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T06:57:32.444Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1854656 }, "sessions": { "total": 1, "taskScoped": 1, "byGezel": { "noor": 1 }, "perTask": { "a-b-test-readout-eval/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.25, "maxMessages": 3, "maxContextFill": 0.2133636474609375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2133636474609375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/ab-test-readout", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/how-we-test-models", "knowledge://bendyline/handboek/craftbook/cohort-analysis", "knowledge://bendyline/handboek/whats-new/1.26273" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "timeout", "reason": "hit hard ceiling (4084720ms) — last hard progress 879s before the end; the deliverable never closed", "durationMs": 4103482, "timeoutMs": 4084720, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-10T06:57:26.409Z", "finishedAt": "2026-11-10T08:05:49.891Z", "timeToFirstArtifactMs": 1862274, "timeToLastArtifactWriteMs": 1862274, "timeToFirstTokenMs": 169444, "firstTurnTtftMs": 152983, "timeToFirstToolCallMs": 205451 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Data Analyst" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 7, "byTool": { "read_file": 2, "write_artifact": 2, "write_task_note": 1, "derive_file": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 7939, "filePath": "craftbook-ab-test-readout", "bytes": 0, "score": 0, "scoreMax": 19, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/experiment-brief.md, data/checkout-test.csv" }, { "atMs": 206213, "filePath": "craftbook-ab-test-readout", "bytes": 0, "score": 1, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" }, { "atMs": 3216800, "filePath": "craftbook-ab-test-readout", "bytes": 521, "score": 11, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" } ], "latest": { "filePath": "craftbook-ab-test-readout", "bytes": 521, "score": 11, "scoreMax": 19, "signals": [], "failReason": "task sourced from craftbook ab-test-readout has not finished: its gates have not accepted the result yet" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T06:57:26.415Z] [trial] id=craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1 scenario=craftbook-ab-test-readout model=underdog-saluki-27b-iq2-mix", "[2026-11-10T06:57:26.415Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-ab-test-readout-ZAdsch", "[2026-11-10T06:57:26.415Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T06:57:26.415Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "[2026-11-10T06:57:26.415Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T06:57:26.418Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T06:57:26.422Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T06:57:26.428Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-ab-test-readout-ZAdsch", "[2026-11-10T06:57:26.428Z] [trial] throughput-scaled ceiling: 45m → 68m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T06:57:32.497Z] [trial] daemon spawned pid=50132 port=https://127.0.0.1:64668", "[2026-11-10T06:57:33.592Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T06:57:33.613Z] [trial] meester=bastien", "[2026-11-10T06:57:33.613Z] [trial] running scenario setup hook", "[2026-11-10T06:57:34.225Z] [trial] scenario setup complete", "[2026-11-10T06:57:34.227Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T06:57:34.293Z] [poll] initial digests hard=69461d80b392ee18 soft=248caf9f90fc81f4", "[2026-11-10T06:57:34.396Z] [poll] hard-progress: 69461d80b392ee18 → bb5aa2b58fe52260 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T06:57:39.500Z] [poll] hard-progress: bb5aa2b58fe52260 → 75527db9099c1eaa after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T07:00:52.678Z] [poll] hard-progress: 75527db9099c1eaa → 10ddbef18d063f8f after 193s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=71,stream=79) sniff=1/0B)", "[2026-11-10T07:23:56.680Z] [poll] hard-progress: 10ddbef18d063f8f → 08abad05f5aa7db1 after 1384s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=3611,stream=4350) sniff=1/0B)", "[2026-11-10T07:28:30.437Z] [poll] hard-progress: 08abad05f5aa7db1 → 5f3bb7a5fff4748d after 274s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=4278,stream=5539) sniff=1/0B)", "[2026-11-10T07:43:32.336Z] [poll] hard-progress: 5f3bb7a5fff4748d → 167b9dba88b80692 after 902s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=6,slot=6267,stream=8591) sniff=1/0B)", "[2026-11-10T07:51:03.249Z] [poll] hard-progress: 167b9dba88b80692 → b015faacc1c1fb6f after 451s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=7,slot=7315,stream=10659) sniff=11/521B)", "[2026-11-10T08:05:42.147Z] [poll] hard ceiling reached after 4084720ms", "[2026-11-10T08:05:42.771Z] [capture] dumped 1 session(s)" ] }, "metrics": { "process": { "sampleCount": 686, "peakRssMb": 11876, "peakCpuPercent": 1215.1, "firstAtMs": 0, "lastAtMs": 4088534 }, "gpu": { "available": true, "sampleCount": 686, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 686, "peakUsedMb": 28923, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 31861, "totalOutputTokens": 7757, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 14222, "todayTokensCached": 13140, "todayTokensOut": 714, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 14222, "totalTokensCached": 13140, "totalTokensOut": 714, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 14222, "cachedInputTokens": 13140, "outputTokens": 714, "cost": 0, "durationMs": 1845812, "outputTokensPerSec": 2.700390731768166, "at": "2026-11-10T07:28:28.682Z" }, "lastUpdated": "2026-11-10T07:28:28.682Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T08:05:42.688Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4090059, "meanTokensPerSec": 2.68, "genTokensPerSec": 2.68, "genTokens": 7757, "promptTokensPerSec": 155.14, "peakRssMb": 11876, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 14222, "output": 714 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T06:57:26.409Z", "finishedAt": "2026-11-10T08:05:49.891Z", "timeToFirstArtifactMs": 1862274, "timeToLastArtifactWriteMs": 1862274, "timeToFirstTokenMs": 169444, "firstTurnTtftMs": 152983, "timeToFirstToolCallMs": 205451 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "a-b-test-readout-eval/1", "stepId": "hypothesis-scope", "activatedAt": "2026-11-10T06:57:34.114Z", "completedAt": "2026-11-10T07:28:28.770Z", "ms": 1854656, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "937bc3ca-e0b7-4cae-a22e-10a93248adaf" ] }, { "ref": "a-b-test-readout-eval/1", "stepId": "analyze", "activatedAt": "2026-11-10T07:28:28.771Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "937bc3ca-e0b7-4cae-a22e-10a93248adaf" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T06:57:32.593Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T06:57:32.444Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1854656 }, "sessions": { "total": 1, "taskScoped": 1, "byGezel": { "noor": 1 }, "perTask": { "a-b-test-readout-eval/1": 1 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.25, "maxMessages": 3, "maxContextFill": 0.2133636474609375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2133636474609375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-craftbook-ab-test-readout", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-ab-test-readout", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "scenarioId": "craftbook-ab-test-readout", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-10T16:59:49.701Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/artifacts/a-b-test-readout-eval/tasks/eval/readout.md", "artifactBytes": 0, "artifactSha256": null, "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/ab/ab-test-readout/versions/1.0.6/test.json", "axes": [ { "name": "decision rule", "description": "The recommendation follows every clause of the pre-registered decision rule, so a breached guardrail overrides a significant positive lift." }, { "name": "statistical reporting", "description": "Each arm's rate, the absolute and relative lift, its 95% confidence interval and the p-value are stated with the right units (percentage points vs percent) and match data/stats.json." }, { "name": "honest caveats", "description": "Caveats on power, runtime and novelty are specific rather than boilerplate, and the readout ends with a concrete next action." } ], "scoreAxes": {}, "meanScore": null, "justification": "No declared readout or verdict was produced before the ceiling. Scope correctly preserves both decision clauses and stats correctly rejects shipping for the0.23pp guardrail breach, but a broken normal-CDF approximation emits p=0. Two derive_file attempts fail; the second still writes stats.json.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/artifacts/a-b-test-readout-eval/tasks/eval/readout.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/workspace/a-b-test-readout-eval/data/stats.json", "status": "notApplicable", "bytes": 521, "sha256": "fa1cae1885ba0eb4c7c636b4289e450ecee3742fd84775ddaad710d879ae51b9", "reviewedBytes": 521, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/artifacts/a-b-test-readout-eval/tasks/eval/hypothesis-scope.md", "status": "notApplicable", "bytes": 1758, "sha256": "4bdced01e014a4510a6c0438ea3667a436e8cfd8f44ca06593d5ee0a126ca6f0", "reviewedBytes": 1758, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/artifacts/a-b-test-readout-eval/tasks/eval/verdict.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T06:57:40.899Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T06:58:51.308Z INFO [chat] [llama-server] launch {\"pid\":21528,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T08:05:49.169Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-21528-1794297949168\",\"pid\":21528,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":4017860,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 21402, "peakGpuMiB": 10630, "peakProcessRssMiB": 11876, "peakSystemRamMiB": 28923, "decodeTokensPerSec": 2.68, "prefillTokensPerSec": 155.14, "generatedTokens": 7757, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2133636474609375 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:53.026Z", "reviewId": "108", "cellId": "r3-saluki-cuda-productivity-craftbook-ab-test-readout", "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "scenarioId": "craftbook-ab-test-readout", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-ab-test-readout", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "2a2acf0126dfbcd9cb2d719dc070a1d6fc6d36773218be544ac6023bff5c04e0", "evidenceSha256": "e0f8aabe02c5fa3d74d1ebead868d10adc9a6ff307f98cf981a01f39930168eb", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/workspace/a-b-test-readout-eval/data/stats.json", "bytes": 521, "sha256": "fa1cae1885ba0eb4c7c636b4289e450ecee3742fd84775ddaad710d879ae51b9", "shownBytes": 521, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/artifacts/a-b-test-readout-eval/tasks/eval/hypothesis-scope.md", "bytes": 1758, "sha256": "4bdced01e014a4510a6c0438ea3667a436e8cfd8f44ca06593d5ee0a126ca6f0", "shownBytes": 1758, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1\\workspace\\a-b-test-readout-eval\\data\\checkout-test.csv", "bytes": 95, "sha256": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "shownBytes": 95, "truncated": false }, { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1\\workspace\\a-b-test-readout-eval\\source\\experiment-brief.md", "bytes": 504, "sha256": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "shownBytes": 504, "truncated": false } ], "scoreAxes": {}, "justification": "Declared eval paths readout.md and verdict.md were missing at closeout. Partial evidence shows only step-1 hypothesis-scope.md (complete, decision rule stated up front) and a stats.json with ship/no-ship fields. No readout or verdict exists to score. The run was runner-terminal, so no outcome is inferred.", "limitations": "Source-only review of the bounded files. No gate logs, transcripts or runtime traces show why the run ended. I did not verify stats.json (p_value 0 looks like rounding). Nothing here supports a success or failure verdict on the model.", "meanScore": null, "durationMs": 14580, "costUsd": 0.020102, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4227, "cache_read_input_tokens": 700, "output_tokens": 312, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4227, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 312, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 4227, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4227 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 312, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 4227, "webSearchRequests": 0, "costUSD": 0.020102, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4227, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-ab-test-readout / underdog-saluki-27b-iq2-mix\n\n**Composite: 4.7 / 10** (band: framework-gap)\n**Outcome: timeout** (duration: 68m 23s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (4084720ms) — last hard progress 879s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 1862274`; `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":521,\"score\":11,\"scoreMax\":19,\"signals\":[],\"failReason\":\"task sourced from craftbook ab-test-readout has not finished: its gates h...` |\n| Output quality (25%) | 5.8 / 10 | The latest scenario gate reached 57.9% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1862274`; `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":521,\"score\":11,\"scoreMax\":19,\"signals\":[],\"failReason\":\"task sourced from craftbook ab-test-readout has not finished: its gates h...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 1862274`; `facts.outcome.durationMs: 4103482` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 5.8 + 0.2 * 2.5 + 0.15 * 10 = **4.7**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11876 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28923 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.68 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 31861 + 7757 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 4 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1855s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.25 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 21% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"timeout\"`\n- `facts.outcome.reason: \"hit hard ceiling (4084720ms) — last hard progress 879s before the end; the deliverable never closed\"`\n- `facts.timing.timeToFirstArtifactMs: 1862274`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":521,\"score\":11,\"scoreMax\":19,\"signals\":[],\"failReason\":\"task sourced from craftbook ab-test-readout has not finished: its gates h...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 57.9% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 1862274`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":521,\"score\":11,\"scoreMax\":19,\"signals\":[],\"failReason\":\"task sourced from craftbook ab-test-readout has not finished: its gates h...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 7`\n- `facts.timing.timeToFirstArtifactMs: 1862274`\n- `facts.outcome.durationMs: 4103482`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"timeout\"`; `facts.outcome.reason: \"hit hard ceiling (4084720ms) — last hard progress 879s before the end; the deliverable never closed\"`; `facts.timing.timeToFirstArtifactMs: 1862274`; `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":521,\"score\":11,\"scoreMax\":19,\"signals\":[],\"failReason\":\"task sourced from craftbook ab-test-readout has not finished: its gates h...`\n- Quality: The latest scenario gate reached 57.9% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1862274`; `facts.sniff.latest: {\"filePath\":\"craftbook-ab-test-readout\",\"bytes\":521,\"score\":11,\"scoreMax\":19,\"signals\":[],\"failReason\":\"task sourced from craftbook ab-test-readout has not finished: its gates h...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 7`; `facts.timing.timeToFirstArtifactMs: 1862274`; `facts.outcome.durationMs: 4103482`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **4.7/10**, eligible task failure; unchanged. Advisory **noArtifact**. No declared readout or verdict was produced before the ceiling. Scope correctly preserves both decision clauses and stats correctly rejects shipping for the0.23pp guardrail breach, but a broken normal-CDF approximation emits p=0. Two derive_file attempts fail; the second still writes stats.json.\n\n\n\n## Positives\n\nThe pre-registered scope reads the exact brief and CSV, keeps the0.20pp guardrail threshold, and correctly treats a breach as overriding significant positive conversion lift. Rates12%/13.25%, absolute1.25pp, relative10.42%, and0.23pp refund delta are correct.\n\n## Negatives and diagnosis\n\nThe first derive_file call uses require in an ES module and fails. The corrected import executes and writes a JSON object, but derive_file rejects non-tabular output and explicitly says the file WAS written. The model does not advance to the required readout. The normal-CDF polynomial is not valid: at zero it returns1 instead of0.5 and here rounds a nonsensical tiny/negative p-value to0. Correct two-sided pooled z is approximately2.66, p≈0.0078. No native failure or mock-bridge failure appears.\n\n## Performance and fit\n\nDecode 2.68tok/s, prefill 155.14tok/s; 7757generated tokens. Peak GPU 10630/12227MiB, process RSS 11876MiB, whole-system RAM 28923/31865MiB. Context 65536, 1slot, f16; max observed released slot 21402. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing analyst role, select run_nodejs_script for non-tabular statistical documents, test CDF sanity points and inspect written output even when a validator reports failure. Then complete the human readout and verdict.\n\n## Strategic experiment — not applied\n\nOffer a tested deterministic two-proportion statistics helper and distinguish execution success from output-shape rejection in tool results. Validate across different sample sizes, signs and guardrail cases, not this fixture alone.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/facts.json): pass, fixed 8, 114.51minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/facts.json): pass, fixed 9, 80.47minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/facts.json): pass, fixed 8.5, 4.80minutes; other historical baseline; contextual only.\n\nEarlier same-scenario trials produced a readout, while this attempt stalls at statistics despite correct guardrail judgment. Records-intake also failed before its primary output; data-wrangle demonstrates successful deterministic transformation elsewhere. Different tool availability and slower decode limit causal comparison.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nBenchmark a generic compute-then-interpret workflow across AB testing, data-wrangle and records-intake with explicit output schema selection; do not retry this row.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-craftbook-annotated-bibliography", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-annotated-bibliography", "suite": "productivity", "scenario": "craftbook-annotated-bibliography", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T08:08:39.485Z", "finishedAt": "2026-11-10T08:51:15.177Z", "durationMs": 2555692, "success": true, "reason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 283479 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-annotated-bibliography", "bytes": 5984, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6256712822421121 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 16 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 283479 }, { "source": "facts", "path": "outcome.durationMs", "value": 2555692 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 9, "band": "ship-ready" }, "facts": { "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "scenarioId": "craftbook-annotated-bibliography", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 426, "peakRssMb": 14675, "peakCpuPercent": 1208, "firstAtMs": 0, "lastAtMs": 2543412 }, "gpu": { "available": true, "sampleCount": 426, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 426, "peakUsedMb": 31180, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 49121, "totalOutputTokens": 5949, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 49273, "todayTokensCached": 44380, "todayTokensOut": 3547, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 49273, "totalTokensCached": 44380, "totalTokensOut": 3547, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7174, "cachedInputTokens": 6878, "outputTokens": 202, "cost": 0, "durationMs": 257184, "outputTokensPerSec": 2.79702822013614, "at": "2026-11-10T08:51:08.581Z" }, "lastUpdated": "2026-11-10T08:51:08.581Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T08:51:10.656Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2544853, "meanTokensPerSec": 2.74, "genTokensPerSec": 2.74, "genTokens": 5949, "promptTokensPerSec": 162.19, "peakRssMb": 14675, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 49273, "output": 3547 } } } }, "craftbook": { "craftbookId": "annotated-bibliography", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "annotated-bibliography-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T08:08:47.223Z", "completedAt": "2026-11-10T08:13:23.042Z", "ms": 275819, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "6595ed2f-6b18-4373-a234-49d229720ad1" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "research", "activatedAt": "2026-11-10T08:13:23.044Z", "completedAt": "2026-11-10T08:25:48.385Z", "ms": 745341, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "61237d1d-f380-4ff0-a532-a4f2cf75e66c" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "write", "activatedAt": "2026-11-10T08:25:48.386Z", "completedAt": "2026-11-10T08:35:50.715Z", "ms": 602329, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "2a9da03b-6325-4d5b-b78c-02955bb1ecdf" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-10T08:35:50.716Z", "completedAt": "2026-11-10T08:46:48.559Z", "ms": 657843, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "5fbc7f8e-f944-4e01-b213-1bf5e8397b63" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "finish", "activatedAt": "2026-11-10T08:46:48.560Z", "completedAt": "2026-11-10T08:51:08.541Z", "ms": 259981, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "4b69cf1b-6a85-4a7c-abde-de6ebdc89752" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T08:08:45.448Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T08:08:45.324Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 602329 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "bente": 1, "beya": 1, "lusine": 1, "penelope": 1, "sri": 1 }, "perTask": { "annotated-bibliography-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.159881591796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.159881591796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/annotated-bibliography", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/reading-list", "knowledge://bendyline/handboek/gezel-app-page-style-guide", "knowledge://bendyline/handboek/craftbook/cohort-analysis" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-annotated-bibliography passed 9 deterministic craftbook checks", "durationMs": 2555692, "timeoutMs": 4084720, "budgetUsedFraction": 0.6256712822421121 }, "timing": { "startedAt": "2026-11-10T08:08:39.485Z", "finishedAt": "2026-11-10T08:51:15.177Z", "timeToFirstArtifactMs": 283479, "timeToLastArtifactWriteMs": 2288883, "timeToFirstTokenMs": 70337, "firstTurnTtftMs": 55428, "timeToFirstToolCallMs": 89473 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Researcher", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 16, "byTool": { "read_file": 3, "write_artifact": 3, "read_artifact": 5, "write_file": 1, "write_task_note": 2, "advance_task_step": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 7990, "filePath": "craftbook-annotated-bibliography", "bytes": 0, "score": 0, "scoreMax": 9, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/source-cards.md" }, { "atMs": 91308, "filePath": "craftbook-annotated-bibliography", "bytes": 0, "score": 1, "scoreMax": 9, "signals": [], "failReason": "task sourced from craftbook annotated-bibliography has not finished: its gates have not accepted the result yet" }, { "atMs": 1569921, "filePath": "craftbook-annotated-bibliography", "bytes": 5984, "score": 8, "scoreMax": 9, "signals": [], "failReason": "task sourced from craftbook annotated-bibliography has not finished: its gates have not accepted the result yet" }, { "atMs": 2550650, "filePath": "craftbook-annotated-bibliography", "bytes": 5984, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-annotated-bibliography", "bytes": 5984, "score": 9, "scoreMax": 9, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T08:08:39.489Z] [trial] id=craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd scenario=craftbook-annotated-bibliography model=underdog-saluki-27b-iq2-mix", "[2026-11-10T08:08:39.489Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-annotated-bibliography-icx98i", "[2026-11-10T08:08:39.489Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "[2026-11-10T08:08:39.489Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T08:08:39.489Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T08:08:39.492Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T08:08:39.495Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T08:08:39.504Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-annotated-bibliography-icx98i", "[2026-11-10T08:08:39.504Z] [trial] throughput-scaled ceiling: 45m → 68m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T08:08:45.387Z] [trial] daemon spawned pid=50456 port=https://127.0.0.1:64924", "[2026-11-10T08:08:46.738Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T08:08:46.757Z] [trial] meester=imara", "[2026-11-10T08:08:46.757Z] [trial] running scenario setup hook", "[2026-11-10T08:08:47.378Z] [trial] scenario setup complete", "[2026-11-10T08:08:47.380Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T08:08:47.433Z] [poll] initial digests hard=c8054a9d5c64ebcf soft=59486d96f8fceb55", "[2026-11-10T08:08:47.528Z] [poll] hard-progress: c8054a9d5c64ebcf → 941a2034a7109ce5 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T08:08:52.695Z] [poll] hard-progress: 941a2034a7109ce5 → c124f5cbac9715b1 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T08:10:10.859Z] [poll] hard-progress: c124f5cbac9715b1 → 9d00e1ae6cb9d5b6 after 78s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=44,stream=44) sniff=1/0B)", "[2026-11-10T08:13:27.012Z] [poll] hard-progress: 9d00e1ae6cb9d5b6 → d8b3d02ec5b13bfd after 196s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=2,slot=524,stream=952) sniff=1/0B)", "[2026-11-10T08:17:34.835Z] [poll] hard-progress: d8b3d02ec5b13bfd → 6dcbc6e7bd608a14 after 248s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=950,stream=1389) sniff=1/0B)", "[2026-11-10T08:25:50.969Z] [poll] hard-progress: 6dcbc6e7bd608a14 → fb8efdf1cd87f9c6 after 496s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=2233,stream=3865) sniff=1/0B)", "[2026-11-10T08:25:56.972Z] [poll] hard-progress: fb8efdf1cd87f9c6 → 8fb785fc5afc3426 after 6s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=5,slot=2233,stream=3865) sniff=1/0B)", "[2026-11-10T08:27:09.623Z] [poll] hard-progress: 8fb785fc5afc3426 → ab453d6370c4900e after 73s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=7,slot=2262,stream=3906) sniff=1/0B)", "[2026-11-10T08:34:49.462Z] [poll] hard-progress: ab453d6370c4900e → 9d2eb8ecc0e57e41 after 460s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=8,slot=3455,stream=6283) sniff=8/5984B)" ] }, "metrics": { "process": { "sampleCount": 426, "peakRssMb": 14675, "peakCpuPercent": 1208, "firstAtMs": 0, "lastAtMs": 2543412 }, "gpu": { "available": true, "sampleCount": 426, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 426, "peakUsedMb": 31180, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 49121, "totalOutputTokens": 5949, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 5, "todayTokensIn": 49273, "todayTokensCached": 44380, "todayTokensOut": 3547, "todayCost": 0, "totalTurns": 5, "totalTokensIn": 49273, "totalTokensCached": 44380, "totalTokensOut": 3547, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 5 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7174, "cachedInputTokens": 6878, "outputTokens": 202, "cost": 0, "durationMs": 257184, "outputTokensPerSec": 2.79702822013614, "at": "2026-11-10T08:51:08.581Z" }, "lastUpdated": "2026-11-10T08:51:08.581Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T08:51:10.656Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2544853, "meanTokensPerSec": 2.74, "genTokensPerSec": 2.74, "genTokens": 5949, "promptTokensPerSec": 162.19, "peakRssMb": 14675, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 49273, "output": 3547 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T08:08:39.485Z", "finishedAt": "2026-11-10T08:51:15.177Z", "timeToFirstArtifactMs": 283479, "timeToLastArtifactWriteMs": 2288883, "timeToFirstTokenMs": 70337, "firstTurnTtftMs": 55428, "timeToFirstToolCallMs": 89473 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 4, "completed": 5, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "annotated-bibliography-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T08:08:47.223Z", "completedAt": "2026-11-10T08:13:23.042Z", "ms": 275819, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "6595ed2f-6b18-4373-a234-49d229720ad1" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "research", "activatedAt": "2026-11-10T08:13:23.044Z", "completedAt": "2026-11-10T08:25:48.385Z", "ms": 745341, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "61237d1d-f380-4ff0-a532-a4f2cf75e66c" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "write", "activatedAt": "2026-11-10T08:25:48.386Z", "completedAt": "2026-11-10T08:35:50.715Z", "ms": 602329, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "2a9da03b-6325-4d5b-b78c-02955bb1ecdf" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-10T08:35:50.716Z", "completedAt": "2026-11-10T08:46:48.559Z", "ms": 657843, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "5fbc7f8e-f944-4e01-b213-1bf5e8397b63" ] }, { "ref": "annotated-bibliography-eval/1", "stepId": "finish", "activatedAt": "2026-11-10T08:46:48.560Z", "completedAt": "2026-11-10T08:51:08.541Z", "ms": 259981, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "4b69cf1b-6a85-4a7c-abde-de6ebdc89752" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T08:08:45.448Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T08:08:45.324Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 602329 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "bente": 1, "beya": 1, "lusine": 1, "penelope": 1, "sri": 1 }, "perTask": { "annotated-bibliography-eval/1": 5 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.7142857142857143, "maxMessages": 2, "maxContextFill": 0.159881591796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.159881591796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-craftbook-annotated-bibliography", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-annotated-bibliography", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "scenarioId": "craftbook-annotated-bibliography", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T16:59:51.753Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/workspace/annotated-bibliography-eval/bibliography.md", "artifactBytes": 5992, "artifactSha256": "060796e9a1956586e31fd1cb8a941005c7ddb568c8ec5f09e86e794cf9183866", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/an/annotated-bibliography/versions/1.0.4/test.json", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ], "scoreAxes": { "accuracy": 8, "grounding": 6, "structure": 8, "actionability": 7 }, "meanScore": 7.3, "justification": "All six seeded sources, dates and figures survive in clear alphabetical annotations with useful method caveats. Some extrapolations are presented as facts: realistic rural volumes, older-patient prevalence, a published privacy checklist, and causal coaching mechanisms exceed the cards. Reviewer checks the generated packet rather than the seed.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/workspace/annotated-bibliography-eval/bibliography.md", "status": "scored", "bytes": 5992, "sha256": "060796e9a1956586e31fd1cb8a941005c7ddb568c8ec5f09e86e794cf9183866", "reviewedBytes": 5992, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/scope.md", "status": "notApplicable", "bytes": 1836, "sha256": "f341dad756907050a3322074228b8939a0909458fbd4c82e7ee06f4ed623449a", "reviewedBytes": 1836, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/sources.md", "status": "notApplicable", "bytes": 5946, "sha256": "eb5a6a77c40c335d48489f549a3c6ac5ac8ee553fe5f41dedcea98da57d134fb", "reviewedBytes": 5946, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/verdict.md", "status": "notApplicable", "bytes": 1003, "sha256": "504d4817b4cdedbee23d40dc8afd2c6b2ec0a9e7d32f9e5123479793ef0d545c", "reviewedBytes": 1003, "truncated": false, "role": "workflowArtifact" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T08:08:52.539Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T08:08:56.326Z INFO [chat] [llama-server] launch {\"pid\":53576,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T08:51:14.434Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-53576-1794300674434\",\"pid\":53576,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":2538108,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 14098, "peakGpuMiB": 10630, "peakProcessRssMiB": 14675, "peakSystemRamMiB": 31180, "decodeTokensPerSec": 2.74, "prefillTokensPerSec": 162.19, "generatedTokens": 5949, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.159881591796875 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:05:58.911Z", "reviewId": "109", "cellId": "r3-saluki-cuda-productivity-craftbook-annotated-bibliography", "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "scenarioId": "craftbook-annotated-bibliography", "status": "scored", "axes": [ { "name": "accuracy", "description": "Computed figures are correct against the seeded records." }, { "name": "grounding", "description": "Every claim traces to the seeded data — nothing invented." }, { "name": "structure", "description": "Tables and sections make the analysis checkable at a glance." }, { "name": "actionability", "description": "Recommendations follow from the findings and are concretely doable." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-annotated-bibliography", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "39d8bb69786e5a0590b826a19b8734b3984db949f510e9b528dbc6eab9acc2f6", "evidenceSha256": "ae91b3059208130b3c25c7a599e928ef0f947eabf3bc4a2c0bb9b5fe10c48978", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/workspace/annotated-bibliography-eval/bibliography.md", "bytes": 5992, "sha256": "060796e9a1956586e31fd1cb8a941005c7ddb568c8ec5f09e86e794cf9183866", "shownBytes": 5992, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/scope.md", "bytes": 1836, "sha256": "f341dad756907050a3322074228b8939a0909458fbd4c82e7ee06f4ed623449a", "shownBytes": 1836, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/sources.md", "bytes": 5946, "sha256": "eb5a6a77c40c335d48489f549a3c6ac5ac8ee553fe5f41dedcea98da57d134fb", "shownBytes": 5946, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/verdict.md", "bytes": 1003, "sha256": "504d4817b4cdedbee23d40dc8afd2c6b2ec0a9e7d32f9e5123479793ef0d545c", "shownBytes": 1003, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd\\workspace\\annotated-bibliography-eval\\source\\source-cards.md", "bytes": 1294, "sha256": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "shownBytes": 1294, "truncated": false } ], "scoreAxes": { "accuracy": 8, "grounding": 6, "structure": 8, "actionability": 6 }, "justification": "All figures match the source cards (11%, 72h, 900, n=240, 18 pts, 42, 1,600) and the order is correct. Some evaluation claims go beyond the cards: Dunn doesn't report readmissions, Patel isn't a controlled comparison, and Williams \"produced\" a checklist. Structure is uniform and clear. Relevance notes give modest design takeaways.", "limitations": "Source-only review of the shown files. Cards give limited method detail, so I could not verify the inferred critiques. No DOIs or volumes are given, but none were supplied. Nothing was run.", "meanScore": 7, "durationMs": 17658, "costUsd": 0.043414, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 8829, "cache_read_input_tokens": 840, "output_tokens": 801, "output_tokens_details": { "thinking_tokens": 464 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 8829, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 801, "cache_read_input_tokens": 840, "cache_creation_input_tokens": 8829, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 8829 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 801, "cacheReadInputTokens": 840, "cacheCreationInputTokens": 8829, "webSearchRequests": 0, "costUSD": 0.043414, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 464, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 8829, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-annotated-bibliography / underdog-saluki-27b-iq2-mix\n\n**Composite: 9.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 42m 36s; budget used: 62.6%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 283479`; `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":5984,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.6256712822421121`; `facts.toolUse.totalToolCalls: 16`; `facts.timing.timeToFirstArtifactMs: 283479`; `facts.outcome.durationMs: 2555692` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 5 + 0.15 * 10 = **9.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14675 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31180 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.74 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 49121 + 5949 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 602s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.71 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 16% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 283479`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":5984,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.6256712822421121`\n- `facts.toolUse.totalToolCalls: 16`\n- `facts.timing.timeToFirstArtifactMs: 283479`\n- `facts.outcome.durationMs: 2555692`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 283479`; `facts.sniff.latest: {\"filePath\":\"craftbook-annotated-bibliography\",\"bytes\":5984,\"score\":9,\"scoreMax\":9,\"signals\":[],\"failReason\":null}`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.6256712822421121`; `facts.toolUse.totalToolCalls: 16`; `facts.timing.timeToFirstArtifactMs: 283479`; `facts.outcome.durationMs: 2555692`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **9/10**, pass; unchanged. Advisory **7.3/10**. All six seeded sources, dates and figures survive in clear alphabetical annotations with useful method caveats. Some extrapolations are presented as facts: realistic rural volumes, older-patient prevalence, a published privacy checklist, and causal coaching mechanisms exceed the cards. Reviewer checks the generated packet rather than the seed.\n\nExact declared axes: accuracy=8; grounding=6; structure=8; actionability=7.\n\n## Positives\n\nSix complete entries retain11%,72hours,900households,n240,18points,42adults and1600encounters accurately. Summary/evaluation/relevance labels make the work easy to audit. Observational causality, self-reported bandwidth and single-region limits are thoughtfully explained.\n\n## Negatives and diagnosis\n\nCritical evaluation often adds sensible reasoning, but some relevance statements overclaim what the tiny source cards establish. The Patel card does not establish that1600encounters is realistic for rural clinics; Garcia does not quantify older-patient prevalence; Williams is useful for a checklist but not stated to have produced one. The verdict treats matching the generated sources packet as proof of no fabrication. No new citation or numerical result was invented.\n\n## Performance and fit\n\nDecode 2.74tok/s, prefill 162.19tok/s; 5949generated tokens. Peak GPU 10630/12227MiB, process RSS 14675MiB, whole-system RAM 31180/31865MiB. Context 65536, 1slot, f16; max observed released slot 14098. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nHave the existing researcher label interpretation as interpretation and the reviewer compare substantive claims with original source cards, not just the generated packet. Keep metadata gaps explicit rather than filling them.\n\n## Strategic experiment — not applied\n\nMaintain claim-to-source spans through source collection and final review, testing both numeric accuracy and unsupported inference across bibliography, Wikipedia and incident briefs.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/facts.json): pass, fixed 9.5, 15.32minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/facts.json): pass, fixed 9.5, 21.22minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/facts.json): pass, fixed 9.5, 1.78minutes; other historical baseline; contextual only.\n\nThe last same-scenario trials are useful for structural consistency, not proof of stronger research. This trial actually reads the seeded source cards, unlike the failed Wikipedia bridge trial; it still illustrates how plausible extrapolations survive a compliant review.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nA/B an evidence-span review on bibliography, research-to-document and conflict-synthesis; measure unsupported claims separately from fixed completion.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-productivity-craftbook-research-to-document", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-research-to-document", "suite": "productivity", "scenario": "craftbook-research-to-document", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T08:53:56.541Z", "finishedAt": "2026-11-10T09:37:21.944Z", "durationMs": 2605403, "success": true, "reason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 257975 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-research-to-document", "bytes": 1679, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6378412718619636 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 34 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 257975 }, { "source": "facts", "path": "outcome.durationMs", "value": 2605403 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "scenarioId": "craftbook-research-to-document", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 434, "peakRssMb": 15786, "peakCpuPercent": 1217.5, "firstAtMs": 1, "lastAtMs": 2591763 }, "gpu": { "available": true, "sampleCount": 434, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 434, "peakUsedMb": 31419, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 96585, "totalOutputTokens": 4525, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 98452, "todayTokensCached": 97351, "todayTokensOut": 973, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 98452, "totalTokensCached": 97351, "totalTokensOut": 973, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9658, "cachedInputTokens": 9587, "outputTokens": 48, "cost": 0, "durationMs": 255101, "outputTokensPerSec": 2.87792550312261, "at": "2026-11-10T09:37:13.809Z" }, "lastUpdated": "2026-11-10T09:37:13.809Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T09:37:16.028Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2593141, "meanTokensPerSec": 2.69, "genTokensPerSec": 2.69, "genTokens": 4525, "promptTokensPerSec": 123.22, "peakRssMb": 15786, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 98452, "output": 973 } } } }, "craftbook": { "craftbookId": "research-to-document", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "word-document-from-content-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T08:54:04.356Z", "completedAt": "2026-11-10T08:58:14.682Z", "ms": 250326, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "a14fbdb3-83b8-41fb-abd9-39c48f901f6b" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "sources", "activatedAt": "2026-11-10T08:58:14.683Z", "completedAt": "2026-11-10T09:02:30.993Z", "ms": 256310, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "5f3918ce-be01-44c2-b19d-83fdc962fe67" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "write", "activatedAt": "2026-11-10T09:02:30.994Z", "completedAt": "2026-11-10T09:08:11.131Z", "ms": 340137, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "7b01c7b0-f06d-4cdb-81e6-68f79a91de8e" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "publish", "activatedAt": "2026-11-10T09:08:11.132Z", "completedAt": "2026-11-10T09:18:38.297Z", "ms": 627165, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "7b01c7b0-f06d-4cdb-81e6-68f79a91de8e" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-10T09:18:38.298Z", "completedAt": "2026-11-10T09:32:55.449Z", "ms": 857151, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "dc619043-f4b7-41f2-afbc-0223193fe66c" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "finish", "activatedAt": "2026-11-10T09:32:55.451Z", "completedAt": "2026-11-10T09:37:13.772Z", "ms": 258321, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "0db6a030-c1e9-46ff-b5b5-a2ed9f0aeac8" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T08:54:02.717Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T08:54:02.586Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 299229 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "alejandra": 1, "bizunesh": 1, "gunnar": 2, "jaehyun": 1 }, "perTask": { "word-document-from-content-eval/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.306610107421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.306610107421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/project-type/support-desk", "knowledge://bendyline/handboek/craftbook/morning-brief", "knowledge://bendyline/handboek/craftbook/source-quality-audit", "knowledge://bendyline/handboek/project-type/hiring-desk", "knowledge://bendyline/handboek/project-type/vendor-contracts" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-research-to-document passed 14 deterministic craftbook checks", "durationMs": 2605403, "timeoutMs": 4084720, "budgetUsedFraction": 0.6378412718619636 }, "timing": { "startedAt": "2026-11-10T08:53:56.541Z", "finishedAt": "2026-11-10T09:37:21.944Z", "timeToFirstArtifactMs": 257975, "timeToLastArtifactWriteMs": 854621, "timeToFirstTokenMs": 70476, "firstTurnTtftMs": 55219, "timeToFirstToolCallMs": 86821 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Planner", "Researcher", "Copywriter", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 34, "byTool": { "read_file": 7, "list_dir": 1, "list_artifacts": 1, "write_artifact": 2, "read_artifact": 4, "write_file": 1, "write_task_note": 4, "advance_task_step": 4, "list_roots": 2, "convert_document": 1, "inspect_document": 2, "preview_document": 2, "save_artifact": 1, "read_task_notes": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [ { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_090233_word-document-docblocks-artifact-surface/tool-0-img-0.png", "bytes": 19310 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_090233_word-document-docblocks-artifact-surface/tool-0-img-1.png", "bytes": 125526 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_090233_word-document-docblocks-artifact-surface/tool-0-img-2.png", "bytes": 42374 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_090233_word-document-docblocks-artifact-surface/tool-0-img-3.png", "bytes": 94927 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_091840_new-session/tool-0-img-0.png", "bytes": 19310 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_091840_new-session/tool-0-img-1.png", "bytes": 125526 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_091840_new-session/tool-0-img-2.png", "bytes": 42374 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_091840_new-session/tool-0-img-3.png", "bytes": 94927 }, { "path": "artifacts/word-document-from-content-eval/sessions/2026-11-10_091840_new-session/tool-0-img-4.png", "bytes": 72673 } ], "otherFileCount": 6 }, "sniff": { "progression": [ { "atMs": 8153, "filePath": "craftbook-research-to-document", "bytes": 0, "score": 1, "scoreMax": 14, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/brief.md" }, { "atMs": 91599, "filePath": "craftbook-research-to-document", "bytes": 0, "score": 2, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 782637, "filePath": "craftbook-research-to-document", "bytes": 1679, "score": 8, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 1094622, "filePath": "craftbook-research-to-document", "bytes": 1679, "score": 9, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 1148221, "filePath": "craftbook-research-to-document", "bytes": 1679, "score": 10, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 1225278, "filePath": "craftbook-research-to-document", "bytes": 1679, "score": 11, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 1320123, "filePath": "craftbook-research-to-document", "bytes": 1679, "score": 13, "scoreMax": 14, "signals": [], "failReason": "task sourced from craftbook research-to-document has not finished: its gates have not accepted the result yet" }, { "atMs": 2598986, "filePath": "craftbook-research-to-document", "bytes": 1679, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-research-to-document", "bytes": 1679, "score": 14, "scoreMax": 14, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T08:53:56.546Z] [trial] id=craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e scenario=craftbook-research-to-document model=underdog-saluki-27b-iq2-mix", "[2026-11-10T08:53:56.546Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-research-to-document-yalY6u", "[2026-11-10T08:53:56.546Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "[2026-11-10T08:53:56.546Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T08:53:56.548Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T08:53:56.546Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T08:53:56.552Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T08:53:56.560Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-research-to-document-yalY6u", "[2026-11-10T08:53:56.560Z] [trial] throughput-scaled ceiling: 45m → 68m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T08:54:02.639Z] [trial] daemon spawned pid=26640 port=https://127.0.0.1:54615", "[2026-11-10T08:54:03.863Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T08:54:03.881Z] [trial] running scenario setup hook", "[2026-11-10T08:54:03.881Z] [trial] meester=wren", "[2026-11-10T08:54:04.528Z] [trial] scenario setup complete", "[2026-11-10T08:54:04.529Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T08:54:04.623Z] [poll] initial digests hard=2ded93596967fe60 soft=037263e404185147", "[2026-11-10T08:54:04.738Z] [poll] hard-progress: 2ded93596967fe60 → d0262a454e734d11 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T08:54:10.088Z] [poll] hard-progress: d0262a454e734d11 → 3af46092ddc750a1 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T08:55:28.189Z] [poll] hard-progress: 3af46092ddc750a1 → 78268de762fcd441 after 78s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=40,stream=40) sniff=2/0B)", "[2026-11-10T08:55:46.126Z] [poll] hard-progress: 78268de762fcd441 → 73bcc640844c2df9 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=64,stream=68) sniff=2/0B)", "[2026-11-10T08:56:10.162Z] [poll] hard-progress: 73bcc640844c2df9 → b41d42ee12900923 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=97,stream=106) sniff=2/0B)", "[2026-11-10T08:58:15.768Z] [poll] hard-progress: b41d42ee12900923 → fa6dcce12d51a7ba after 126s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=403,stream=685) sniff=2/0B)", "[2026-11-10T08:58:23.687Z] [poll] hard-progress: fa6dcce12d51a7ba → 7495794b7c1e3b98 after 8s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=403,stream=685) sniff=2/0B)", "[2026-11-10T08:59:41.455Z] [poll] hard-progress: 7495794b7c1e3b98 → 1a3f7086d10fb115 after 78s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=456,stream=740) sniff=2/0B)", "[2026-11-10T09:00:11.422Z] [poll] hard-progress: 1a3f7086d10fb115 → 6c5ac2fb7f49485f after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=498,stream=790) sniff=2/0B)" ] }, "metrics": { "process": { "sampleCount": 434, "peakRssMb": 15786, "peakCpuPercent": 1217.5, "firstAtMs": 1, "lastAtMs": 2591763 }, "gpu": { "available": true, "sampleCount": 434, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 434, "peakUsedMb": 31419, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 96585, "totalOutputTokens": 4525, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 98452, "todayTokensCached": 97351, "todayTokensOut": 973, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 98452, "totalTokensCached": 97351, "totalTokensOut": 973, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9658, "cachedInputTokens": 9587, "outputTokens": 48, "cost": 0, "durationMs": 255101, "outputTokensPerSec": 2.87792550312261, "at": "2026-11-10T09:37:13.809Z" }, "lastUpdated": "2026-11-10T09:37:13.809Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T09:37:16.028Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2593141, "meanTokensPerSec": 2.69, "genTokensPerSec": 2.69, "genTokens": 4525, "promptTokensPerSec": 123.22, "peakRssMb": 15786, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 98452, "output": 973 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T08:53:56.541Z", "finishedAt": "2026-11-10T09:37:21.944Z", "timeToFirstArtifactMs": 257975, "timeToLastArtifactWriteMs": 854621, "timeToFirstTokenMs": 70476, "firstTurnTtftMs": 55219, "timeToFirstToolCallMs": 86821 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 4, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "word-document-from-content-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T08:54:04.356Z", "completedAt": "2026-11-10T08:58:14.682Z", "ms": 250326, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "a14fbdb3-83b8-41fb-abd9-39c48f901f6b" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "sources", "activatedAt": "2026-11-10T08:58:14.683Z", "completedAt": "2026-11-10T09:02:30.993Z", "ms": 256310, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "5f3918ce-be01-44c2-b19d-83fdc962fe67" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "write", "activatedAt": "2026-11-10T09:02:30.994Z", "completedAt": "2026-11-10T09:08:11.131Z", "ms": 340137, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "7b01c7b0-f06d-4cdb-81e6-68f79a91de8e" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "publish", "activatedAt": "2026-11-10T09:08:11.132Z", "completedAt": "2026-11-10T09:18:38.297Z", "ms": 627165, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "7b01c7b0-f06d-4cdb-81e6-68f79a91de8e" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "evaluate", "activatedAt": "2026-11-10T09:18:38.298Z", "completedAt": "2026-11-10T09:32:55.449Z", "ms": 857151, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "dc619043-f4b7-41f2-afbc-0223193fe66c" ] }, { "ref": "word-document-from-content-eval/1", "stepId": "finish", "activatedAt": "2026-11-10T09:32:55.451Z", "completedAt": "2026-11-10T09:37:13.772Z", "ms": 258321, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "0db6a030-c1e9-46ff-b5b5-a2ed9f0aeac8" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T08:54:02.717Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T08:54:02.586Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 299229 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "alejandra": 1, "bizunesh": 1, "gunnar": 2, "jaehyun": 1 }, "perTask": { "word-document-from-content-eval/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.306610107421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.306610107421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-productivity-craftbook-research-to-document", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-research-to-document", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "scenarioId": "craftbook-research-to-document", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T16:59:53.654Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/workspace/word-document-from-content-eval/documents/eval/report.md", "artifactBytes": 1691, "artifactSha256": "c9625a9a378527befca2070b6e91770c99649fbcce3a7982d3529fea5792392b", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/re/research-to-document/versions/1.2.8/test.json", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ], "scoreAxes": { "grounding": 6, "structure": 8, "convertibility": 8 }, "meanScore": 7.3, "justification": "Accurate source values and actions form a clear five-section report with a real saved DOCX and intact table. “Leakage nearly halved” overstates the37.3% relative reduction, and audience priorities are invented. All five reconstructed previews are readable; native pagination/fonts remain explicitly unverified.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/workspace/word-document-from-content-eval/documents/eval/report.md", "status": "scored", "bytes": 1691, "sha256": "c9625a9a378527befca2070b6e91770c99649fbcce3a7982d3529fea5792392b", "reviewedBytes": 1691, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/artifacts/word-document-from-content-eval/tasks/eval/scope.md", "status": "notApplicable", "bytes": 1143, "sha256": "a103153d9c52befe23dce5b02ccc62bdced1b229f3ea19d2440a343a920e3c64", "reviewedBytes": 1143, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/artifacts/word-document-from-content-eval/tasks/eval/sources.md", "status": "notApplicable", "bytes": 1094, "sha256": "13efb28df29f1991b040effb818998876f341405f685e865c5dc84280c163c58", "reviewedBytes": 1094, "truncated": false, "role": "workflowArtifact" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/artifacts/word-document-from-content-eval/tasks/eval/report.docx", "status": "notApplicable", "bytes": 18958, "sha256": "b0a855581015a90eb375f8219ce8b03c26af48e1b6c5686647a5ce981c5c6099", "reviewedBytes": 18958, "truncated": false, "role": "supportingBinary" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T08:54:09.844Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T08:54:13.731Z INFO [chat] [llama-server] launch {\"pid\":28076,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T09:37:21.104Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-28076-1794303441104\",\"pid\":28076,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":2587374,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 31459, "peakGpuMiB": 10646, "peakProcessRssMiB": 15786, "peakSystemRamMiB": 31419, "decodeTokensPerSec": 2.69, "prefillTokensPerSec": 123.22, "generatedTokens": 4525, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.306610107421875 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:09.605Z", "reviewId": "110", "cellId": "r3-saluki-cuda-productivity-craftbook-research-to-document", "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "scenarioId": "craftbook-research-to-document", "status": "scored", "axes": [ { "name": "grounding", "description": "Every metric and action comes from the seeded brief." }, { "name": "structure", "description": "The source has a usable title, summary, sections, table, and next actions." }, { "name": "convertibility", "description": "The Markdown is complete enough for direct DocBlocks DOCX conversion without HTML or XML." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "baselineCellId": "r1-saluki-cuda-productivity-craftbook-research-to-document", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "99f95050ddf84ea3765fe65f551cc7d9b33a5e5ad67f7b9b8d2f8ba0248c0c54", "evidenceSha256": "33557bad3cfed3019e3e42d177bb3b7ee90f4689d4ff46c5b88bf67b4899cdbb", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/workspace/word-document-from-content-eval/documents/eval/report.md", "bytes": 1691, "sha256": "c9625a9a378527befca2070b6e91770c99649fbcce3a7982d3529fea5792392b", "shownBytes": 1691, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/artifacts/word-document-from-content-eval/tasks/eval/scope.md", "bytes": 1143, "sha256": "a103153d9c52befe23dce5b02ccc62bdced1b229f3ea19d2440a343a920e3c64", "shownBytes": 1143, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/artifacts/word-document-from-content-eval/tasks/eval/sources.md", "bytes": 1094, "sha256": "13efb28df29f1991b040effb818998876f341405f685e865c5dc84280c163c58", "shownBytes": 1094, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e\\workspace\\word-document-from-content-eval\\source\\brief.md", "bytes": 405, "sha256": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "shownBytes": 405, "truncated": false } ], "scoreAxes": { "grounding": 6, "structure": 9, "convertibility": 8 }, "justification": "All core metrics (18 SKUs, 18h→6h, 14.2%→8.9%) and the three next actions match brief.md. But \"nearly halved\" overstates a 37% drop, \"care about most\" is invented, and the action rationales (barcode errors, Finance view) are inferred. Structure has title, summary, 3-row table, findings, numbered actions. Plain Markdown, no HTML.", "limitations": "Source-only review. No DOCX conversion or preview was seen, so convertibility is judged from the Markdown syntax alone. Whether DocBlocks renders the table cleanly was not verified.", "meanScore": 7.666666666666667, "durationMs": 16481, "costUsd": 0.023922399999999996, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4377, "cache_read_input_tokens": 804, "output_tokens": 633, "output_tokens_details": { "thinking_tokens": 297 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4377, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 633, "cache_read_input_tokens": 804, "cache_creation_input_tokens": 4377, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4377 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 633, "cacheReadInputTokens": 804, "cacheCreationInputTokens": 4377, "webSearchRequests": 0, "costUSD": 0.023922399999999996, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 297, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4377, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-research-to-document / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 43m 25s; budget used: 63.8%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 257975`; `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1679,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.6378412718619636`; `facts.toolUse.totalToolCalls: 34`; `facts.timing.timeToFirstArtifactMs: 257975`; `facts.outcome.durationMs: 2605403` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15786 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31419 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.69 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 96585 + 4525 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 8 → 6 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 299s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.63 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 31% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 257975`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1679,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.6378412718619636`\n- `facts.toolUse.totalToolCalls: 34`\n- `facts.timing.timeToFirstArtifactMs: 257975`\n- `facts.outcome.durationMs: 2605403`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 257975`; `facts.sniff.latest: {\"filePath\":\"craftbook-research-to-document\",\"bytes\":1679,\"score\":14,\"scoreMax\":14,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.6378412718619636`; `facts.toolUse.totalToolCalls: 34`; `facts.timing.timeToFirstArtifactMs: 257975`; `facts.outcome.durationMs: 2605403`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.5/10**, pass; unchanged. Advisory **7.3/10**. Accurate source values and actions form a clear five-section report with a real saved DOCX and intact table. “Leakage nearly halved” overstates the37.3% relative reduction, and audience priorities are invented. All five reconstructed previews are readable; native pagination/fonts remain explicitly unverified.\n\nExact declared axes: grounding=6; structure=8; convertibility=8.\n\n## Positives\n\nThe scope and source log preserve the single-source boundary.18SKUs,18→6hours and14.2→8.9% appear correctly in the metrics table; all three supplied next actions remain. Real convert/save/inspect receipts confirm an editable DOCX with one4×3table. The reviewer candidly distinguishes reconstructed preview from native pixels.\n\n## Negatives and diagnosis\n\nThe findings use an overstated near-halving phrase for a5.3/14.2=37.3% relative reduction. The source does not say these are the two metrics operations leaders care about most. Derived12hours/5.3points are arithmetically correct but conflict with the claim that all figures are verbatim. The reviewer misses these content issues despite reading the source. Exact native Word layout is not established by normalized previews.\n\n## Performance and fit\n\nDecode 2.69tok/s, prefill 123.22tok/s; 4525generated tokens. Peak GPU 10646/12227MiB, process RSS 15786MiB, whole-system RAM 31419/31865MiB. Context 65536, 1slot, f16; max observed released slot 31459. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nRequire the existing document reviewer to recompute comparative language and distinguish quoted values, derived values and interpretation. Preserve the useful explicit preview limitation.\n\n## Strategic experiment — not applied\n\nSeparate content-grounding review from file-format/preview validation so a clean conversion cannot imply factual fidelity. Test on slide, spreadsheet and prose outputs.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/facts.json): pass, fixed 8.5, 25.83minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/facts.json): pass, fixed 8.5, 19.63minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/facts.json): pass, fixed 8.5, 2.56minutes; other historical baseline; contextual only.\n\nEarlier research-to-document trials and the PowerPoint trial share the same small fixture metrics; compare interpretation errors, not just artifact presence. Theme-roundtrip illustrates a weaker response to reported degradation; this review is more candid about preview limits but still misses unsupported prose.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nEvaluate generic ratio-language and source-boundary review across research-to-document, PowerPoint-deck and AB readout before changing any role.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-fix-squisq-bugs", "baselineCellId": "r1-saluki-cuda-developer-fix-squisq-bugs", "suite": "developer", "scenario": "fix-squisq-bugs", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T09:40:09.309Z", "finishedAt": "2026-11-10T10:04:35.584Z", "durationMs": 1466275, "success": false, "reason": "no real progress for 1363s — model busy but not delivering (hard digest stuck at 6e03812b5ef71586)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "fix-squisq-bugs", "score": 1, "bytes": 5461 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1363s — model busy but not delivering (hard digest stuck at 6e03812b5ef71586)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "fix-squisq-bugs", "bytes": 5461, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng 87.8 — it must cross the antimeridian (±180), not run back through 0" } } ] }, "quality": { "score": 2, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 20% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "fix-squisq-bugs", "bytes": 5461, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng 87.8 — it must cross the antimeridian (±180), not run back through 0" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6461385041746556 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 1466275 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.5, "band": "framework-gap" }, "facts": { "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "scenarioId": "fix-squisq-bugs", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 245, "peakRssMb": 9425, "peakCpuPercent": 1204.9, "firstAtMs": 0, "lastAtMs": 1453319 }, "gpu": { "available": true, "sampleCount": 245, "peakUtilPercent": 100, "peakMemUsedMb": 10624, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 245, "peakUsedMb": 26416, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 3754, "totalOutputTokens": 142, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T10:04:30.436Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1454796, "meanTokensPerSec": 2.99, "genTokensPerSec": 2.99, "genTokens": 142, "promptTokensPerSec": 162.65, "peakRssMb": 9425, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T09:40:15.198Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T09:40:15.070Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "dani": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.051361083984375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.051361083984375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1363s — model busy but not delivering (hard digest stuck at 6e03812b5ef71586)", "durationMs": 1466275, "timeoutMs": 2269289, "budgetUsedFraction": 0.6461385041746556 }, "timing": { "startedAt": "2026-11-10T09:40:09.309Z", "finishedAt": "2026-11-10T10:04:35.584Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 47075, "firstTurnTtftMs": 35539, "timeToFirstToolCallMs": 94620 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 4, "byTool": { "read_file": 4 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 8156, "filePath": "fix-squisq-bugs", "bytes": 5461, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng 87.8 — it must cross the antimeridian (±180), not run back through 0" } ], "latest": { "filePath": "fix-squisq-bugs", "bytes": 5461, "score": 1, "scoreMax": 5, "signals": [ "haversine-untouched" ], "failReason": "getGeohashPath behavioral check failed: path from lng 170 to lng -170 passes through lng 87.8 — it must cross the antimeridian (±180), not run back through 0" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T09:40:09.312Z] [trial] id=fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t scenario=fix-squisq-bugs model=underdog-saluki-27b-iq2-mix", "[2026-11-10T09:40:09.312Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-fix-squisq-bugs-pp2nG2", "[2026-11-10T09:40:09.312Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T09:40:09.312Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "[2026-11-10T09:40:09.313Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T09:40:09.315Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T09:40:09.320Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T09:40:09.326Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-fix-squisq-bugs-pp2nG2", "[2026-11-10T09:40:09.326Z] [trial] throughput-scaled ceiling: 25m → 38m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T09:40:15.101Z] [trial] daemon spawned pid=28904 port=https://127.0.0.1:63127", "[2026-11-10T09:40:16.583Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T09:40:16.601Z] [trial] meester=zara", "[2026-11-10T09:40:16.601Z] [trial] running scenario setup hook", "[2026-11-10T09:40:17.121Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T09:40:17.120Z] [trial] scenario setup complete", "[2026-11-10T09:40:17.211Z] [poll] initial digests hard=6ef9aae81c403afb soft=07c1be81591b51e8", "[2026-11-10T09:40:17.584Z] [poll] hard-progress: 6ef9aae81c403afb → 9929e1d876507434 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/5461B)", "[2026-11-10T09:41:46.616Z] [poll] hard-progress: 9929e1d876507434 → 6e03812b5ef71586 after 89s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=69,stream=107) sniff=1/5461B)", "[2026-11-10T10:04:29.953Z] [poll] no-progress (hard): no real progress for 1363s (threshold=1362s); model busy but not delivering — failing trial", "[2026-11-10T10:04:30.481Z] [capture] dumped 1 session(s)", "[2026-11-10T10:04:30.484Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-10T10:04:30.484Z] [capture] INCOMPLETE TRANSCRIPT for 1 session(s) — dani: 0 persisted vs 4 recorded. Assistant turns commit at turn END, so a mid-turn termination loses them. Triage from session-telemetry.json and daemon.log, NOT from the empty transcript.", "[2026-11-10T10:04:30.549Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-10T10:04:35.583Z] [trial] daemon shut down", "[2026-11-10T10:04:35.595Z] [trial] FAIL duration=1466275ms reason=no real progress for 1363s — model busy but not delivering (hard digest stuck at 6e03812b5ef71586)" ] }, "metrics": { "process": { "sampleCount": 245, "peakRssMb": 9425, "peakCpuPercent": 1204.9, "firstAtMs": 0, "lastAtMs": 1453319 }, "gpu": { "available": true, "sampleCount": 245, "peakUtilPercent": 100, "peakMemUsedMb": 10624, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 245, "peakUsedMb": 26416, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 3754, "totalOutputTokens": 142, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T10:04:30.436Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1454796, "meanTokensPerSec": 2.99, "genTokensPerSec": 2.99, "genTokens": 142, "promptTokensPerSec": 162.65, "peakRssMb": 9425, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T09:40:09.309Z", "finishedAt": "2026-11-10T10:04:35.584Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 47075, "firstTurnTtftMs": 35539, "timeToFirstToolCallMs": 94620 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T09:40:15.198Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T09:40:15.070Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "dani": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.051361083984375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.051361083984375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-fix-squisq-bugs", "baselineCellId": "r1-saluki-cuda-developer-fix-squisq-bugs", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "scenarioId": "fix-squisq-bugs", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-10T17:01:33.025Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "artifactBytes": 3830, "artifactSha256": "29c40c6c7649ec7d2794d514fd31f9b422563d5a2076e1951737983ad89dc4e9", "axisSource": "evals/src/scenarios/fix-squisq-bugs.ts; no declared axes", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "Four file reads produced no edit or authored artifact. The seeded linear longitude interpolation, shallow layer validation and invalid-length throws remain. Haversine was correctly left untouched. The runner stalls with1/5 checks; no bridge or native failure is evident.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "status": "notApplicable", "bytes": 3830, "sha256": "29c40c6c7649ec7d2794d514fd31f9b422563d5a2076e1951737983ad89dc4e9", "reviewedBytes": 3830, "truncated": false, "role": "unchangedSeededTarget" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/imageEdit/persistence.ts", "status": "notApplicable", "bytes": 1633, "sha256": "93ae10d0b079c355a29b52b1d2af83999dfe4fecd38d6309b795c6987525c3df", "reviewedBytes": 1633, "truncated": false, "role": "unchangedSeededTarget" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/spatial/Haversine.ts", "status": "notApplicable", "bytes": 749, "sha256": "6ba31700e271f0f3a33aff0339cb30e902bb890c78d74bb9c47f2e4b06eac635", "reviewedBytes": 749, "truncated": false, "role": "protectedSeededSource" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T09:40:18.996Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T09:40:24.128Z INFO [chat] [llama-server] launch {\"pid\":37092,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T10:04:34.821Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-37092-1794305074821\",\"pid\":37092,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":1450694,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 11035, "peakGpuMiB": 10624, "peakProcessRssMiB": 9425, "peakSystemRamMiB": 26416, "decodeTokensPerSec": 2.99, "prefillTokensPerSec": 162.65, "generatedTokens": 142, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.051361083984375 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:13.218Z", "reviewId": "111", "cellId": "r3-saluki-cuda-developer-fix-squisq-bugs", "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "scenarioId": "fix-squisq-bugs", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "baselineCellId": "r1-saluki-cuda-developer-fix-squisq-bugs", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "37170962453dda8642dd6214cb7f4bd47981850345119ea1e381e184aa6baf38", "evidenceSha256": "bf8a2e5255b1e866137ad61e8dd83ef2469edb7424f8fce959ba88c55df48984", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/spatial/Geohash.ts", "bytes": 3830, "sha256": "29c40c6c7649ec7d2794d514fd31f9b422563d5a2076e1951737983ad89dc4e9", "shownBytes": 3830, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/imageEdit/persistence.ts", "bytes": 1633, "sha256": "93ae10d0b079c355a29b52b1d2af83999dfe4fecd38d6309b795c6987525c3df", "shownBytes": 1633, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/spatial/Haversine.ts", "bytes": 749, "sha256": "6ba31700e271f0f3a33aff0339cb30e902bb890c78d74bb9c47f2e4b06eac635", "shownBytes": 749, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No deliverable to score. The snapshot shows Geohash.ts and persistence.ts still unfixed: getGeohashPath interpolates linearly, assertImageEditDoc has no per-layer checks, and both geohash functions still throw on bad length. Bug-report comments remain. Haversine.ts is unchanged. No declared paths are missing, but no edits are visible.", "limitations": "Only a final-state source snapshot was supplied. There is no transcript, tool log or tsc output, so I can't tell why the edits are absent (interruption, no attempt, or capture failure). This is not a verdict on model capability, and no runtime was run.", "meanScore": null, "durationMs": 14593, "costUsd": 0.022851999999999997, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 4872, "cache_read_input_tokens": 700, "output_tokens": 329, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 4872, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 329, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 4872, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4872 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 329, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 4872, "webSearchRequests": 0, "costUSD": 0.022851999999999997, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 4872, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - fix-squisq-bugs / underdog-saluki-27b-iq2-mix\n\n**Composite: 3.5 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 24m 26s; budget used: 64.6%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1363s — model busy but not delivering (hard digest stuck at 6e03812b5ef71586)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":5461,\"score\":1,\"scoreMax\":5,\"signals\":[\"haversine-untouched\"],\"failReason\":\"getGeohashPath behavioral check failed: path from lng 170 to ln...` |\n| Output quality (25%) | 2 / 10 | The latest scenario gate reached 20% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":5461,\"score\":1,\"scoreMax\":5,\"signals\":[\"haversine-untouched\"],\"failReason\":\"getGeohashPath behavioral check failed: path from lng 170 to ln...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.6461385041746556`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 1466275` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 2 + 0.2 * 7.5 + 0.15 * 10 = **3.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9425 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10624 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26416 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.99 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 3754 + 142 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 5% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1363s — model busy but not delivering (hard digest stuck at 6e03812b5ef71586)\"`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":5461,\"score\":1,\"scoreMax\":5,\"signals\":[\"haversine-untouched\"],\"failReason\":\"getGeohashPath behavioral check failed: path from lng 170 to ln...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 20% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":5461,\"score\":1,\"scoreMax\":5,\"signals\":[\"haversine-untouched\"],\"failReason\":\"getGeohashPath behavioral check failed: path from lng 170 to ln...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.6461385041746556`\n- `facts.toolUse.totalToolCalls: 4`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 1466275`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1363s — model busy but not delivering (hard digest stuck at 6e03812b5ef71586)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":5461,\"score\":1,\"scoreMax\":5,\"signals\":[\"haversine-untouched\"],\"failReason\":\"getGeohashPath behavioral check failed: path from lng 170 to ln...`\n- Quality: The latest scenario gate reached 20% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"fix-squisq-bugs\",\"bytes\":5461,\"score\":1,\"scoreMax\":5,\"signals\":[\"haversine-untouched\"],\"failReason\":\"getGeohashPath behavioral check failed: path from lng 170 to ln...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.6461385041746556`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 1466275`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **3.5/10**, eligible task failure; unchanged. Advisory **noArtifact**. Four file reads produced no edit or authored artifact. The seeded linear longitude interpolation, shallow layer validation and invalid-length throws remain. Haversine was correctly left untouched. The runner stalls with1/5 checks; no bridge or native failure is evident.\n\n\n\n## Positives\n\nThe model reads the small explicit bug report and named sources. It respects the instruction not to modify the correct Haversine helper.\n\n## Negatives and diagnosis\n\nNo implementation follows the reads despite write/replace tools being present. The antimeridian behavioral check still traverses87.8degrees instead of the shortest arc. All three described fixes remain undone. Existing target files are preserved seed inputs, not produced artifacts; advisory status is noArtifact.\n\n## Performance and fit\n\nDecode 2.99tok/s, prefill 162.65tok/s; 142generated tokens. Peak GPU 10624/12227MiB, process RSS 9425MiB, whole-system RAM 26416/31865MiB. Context 65536, 1slot, f16; max observed released slot 11035. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing developer role, make one minimal verifiable edit after reading the named defect, then proceed through the remaining checklist. Record progress as changes rather than extended analysis.\n\n## Strategic experiment — not applied\n\nDetect read-only stagnation with actionable next-edit feedback while keeping the watchdog outcome separate from load health. Validate on several repair sizes and numeric/string transformations.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/facts.json): pass, fixed 10, 8.71minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/facts.json): pass, fixed 9.5, 14.38minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/facts.json): pass, fixed 8.5, 7.59minutes; other historical baseline; contextual only.\n\nPrevious same-scenario trials authored repairs; this attempt never reaches a first edit. Interface-contract succeeds with three small writes, while other developer workflows also stall after scoping. Slower2.99tok/s does not explain a particular lack of output by itself.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nCompare a generic bounded read-plan-edit workflow across squisq repairs, schema migration and failing tests, without fixture-specific solutions or retrying this row.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-dev-craftbook-routing", "baselineCellId": "r1-saluki-cuda-developer-dev-craftbook-routing", "suite": "developer", "scenario": "dev-craftbook-routing", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T10:06:52.677Z", "finishedAt": "2026-11-10T10:10:47.646Z", "durationMs": 234969, "success": true, "reason": "task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.10354300399816858 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 1 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 234969 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" }, "facts": { "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "scenarioId": "dev-craftbook-routing", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 38, "peakRssMb": 9660, "peakCpuPercent": 1207.6, "firstAtMs": 0, "lastAtMs": 221637 }, "gpu": { "available": true, "sampleCount": 38, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 38, "peakUsedMb": 26731, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11820, "totalOutputTokens": 347, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T10:10:42.105Z" } }, "source": "engine-log" }, "derived": { "durationMs": 223007, "meanTokensPerSec": 2.81, "genTokensPerSec": 2.81, "genTokens": 347, "promptTokensPerSec": 168.92, "peakRssMb": 9660, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "batch-exporter/1", "stepId": "reproduce", "activatedAt": "2026-11-10T10:10:39.249Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T10:06:58.604Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T10:06:58.476Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "imara": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.153564453125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.153564453125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "durationMs": 234969, "timeoutMs": 2269289, "budgetUsedFraction": 0.10354300399816858 }, "timing": { "startedAt": "2026-11-10T10:06:52.677Z", "finishedAt": "2026-11-10T10:10:47.646Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 102009, "firstTurnTtftMs": 81389, "timeToFirstToolCallMs": 226755 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 1, "byTool": { "start_work": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 3 }, "sniff": { "progression": [ { "atMs": 8045, "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } ], "latest": { "filePath": "dev-craftbook-routing", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task has been started from a library recipe yet — search the recipe library for the one that matches this brief (the cause is known, nothing is urgent, and a failing test is wanted before the fix) and start the task from it" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T10:06:52.682Z] [trial] id=dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb scenario=dev-craftbook-routing model=underdog-saluki-27b-iq2-mix", "[2026-11-10T10:06:52.682Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-dev-craftbook-routing-feXvKi", "[2026-11-10T10:06:52.682Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T10:06:52.682Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T10:06:52.682Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "[2026-11-10T10:06:52.686Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T10:06:52.690Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T10:06:52.696Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-dev-craftbook-routing-feXvKi", "[2026-11-10T10:06:52.697Z] [trial] throughput-scaled ceiling: 25m → 38m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T10:06:58.532Z] [trial] daemon spawned pid=7176 port=https://127.0.0.1:52279", "[2026-11-10T10:07:00.024Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T10:07:00.041Z] [trial] meester=imara", "[2026-11-10T10:07:00.041Z] [trial] running scenario setup hook", "[2026-11-10T10:07:00.370Z] [trial] scenario setup complete", "[2026-11-10T10:07:00.371Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T10:07:00.450Z] [poll] initial digests hard=b08154bd588febfe soft=5775124edcac9630", "[2026-11-10T10:07:00.827Z] [poll] hard-progress: b08154bd588febfe → ff9ef067a257dd02 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/3B)", "[2026-11-10T10:10:41.938Z] [poll] terminal: success=true reason=task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored", "[2026-11-10T10:10:42.143Z] [capture] dumped 1 session(s)", "[2026-11-10T10:10:42.146Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-10T10:10:42.203Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-10T10:10:47.645Z] [trial] daemon shut down", "[2026-11-10T10:10:47.655Z] [trial] PASS duration=234969ms reason=task batch-exporter/1 was started from the bundled \"bug-fix-tdd\" recipe with no new craftbook authored" ] }, "metrics": { "process": { "sampleCount": 38, "peakRssMb": 9660, "peakCpuPercent": 1207.6, "firstAtMs": 0, "lastAtMs": 221637 }, "gpu": { "available": true, "sampleCount": 38, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 38, "peakUsedMb": 26731, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11820, "totalOutputTokens": 347, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T10:10:42.105Z" } }, "source": "engine-log" }, "derived": { "durationMs": 223007, "meanTokensPerSec": 2.81, "genTokensPerSec": 2.81, "genTokens": 347, "promptTokensPerSec": 168.92, "peakRssMb": 9660, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T10:06:52.677Z", "finishedAt": "2026-11-10T10:10:47.646Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 102009, "firstTurnTtftMs": 81389, "timeToFirstToolCallMs": 226755 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "batch-exporter/1", "stepId": "reproduce", "activatedAt": "2026-11-10T10:10:39.249Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T10:06:58.604Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T10:06:58.476Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "imara": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.153564453125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.153564453125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-dev-craftbook-routing", "baselineCellId": "r1-saluki-cuda-developer-dev-craftbook-routing", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "scenarioId": "dev-craftbook-routing", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-10T17:01:34.862Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/recording/chat-events.jsonl", "artifactBytes": 194013, "artifactSha256": "943b9307f7f328b878fb5fca2f137c3b2ca7d41d73d842759ad14b8903dec35b", "axisSource": "evals/src/scenarios/dev-craftbook-routing.ts; no declared axes", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "One start_work call correctly selects bundled bug-fix-tdd and dispatches the reproduction step. This scenario tests routing only; no repaired code or completed regression-test workflow is claimed. There are no declared artifact axes.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/recording/chat-events.jsonl", "status": "notApplicable", "bytes": 194013, "sha256": "943b9307f7f328b878fb5fca2f137c3b2ca7d41d73d842759ad14b8903dec35b", "reviewedBytes": 20480, "truncated": true, "role": "routingEvidence" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T10:07:02.245Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T10:07:16.965Z INFO [chat] [llama-server] launch {\"pid\":31092,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T10:10:46.967Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-31092-1794305446967\",\"pid\":31092,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":210003,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 12166, "peakGpuMiB": 10628, "peakProcessRssMiB": 9660, "peakSystemRamMiB": 26731, "decodeTokensPerSec": 2.81, "prefillTokensPerSec": 168.92, "generatedTokens": 347, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.153564453125 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:13.581Z", "reviewId": "112", "cellId": "r3-saluki-cuda-developer-dev-craftbook-routing", "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "scenarioId": "dev-craftbook-routing", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "baselineCellId": "r1-saluki-cuda-developer-dev-craftbook-routing", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "b936d60a8e8bcbdd52eff76ea2d73ff9f4f44c3f35d495fb51f8c0db3fda7738", "evidenceSha256": "f64d165204999a6923cd1980c634fff43c3cf7669b1278827a4b224b974bbaff", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/recording/chat-events.jsonl", "bytes": 194013, "sha256": "943b9307f7f328b878fb5fca2f137c3b2ca7d41d73d842759ad14b8903dec35b", "shownBytes": 20480, "truncated": true } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. The visible 20 KB of chat-events shows only the user prompt, engine prefill and early reasoning (\"set up a task in this project using the recipe library\"). It shows no recipe listing, no task start, and no source edits.", "limitations": "The artifact is truncated (20 of 194 KB). I could not see recipe selection, task creation, or whether the run avoided start_project and source edits. Nothing can be concluded about success or failure.", "meanScore": null, "durationMs": 14573, "costUsd": 0.065492, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 15687, "cache_read_input_tokens": 700, "output_tokens": 267, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 15687, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 267, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 15687, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 15687 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 267, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 15687, "webSearchRequests": 0, "costUSD": 0.065492, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 15687, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - dev-craftbook-routing / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.8 / 10** (band: ship-ready)\n**Outcome: success** (duration: 3m 55s; budget used: 10.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 5 / 10 | The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.10354300399816858`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 234969` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **8.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9660 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26731 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.81 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 11820 + 347 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 50% of its measured signals.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.10354300399816858`\n- `facts.toolUse.totalToolCalls: 1`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 234969`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"dev-craftbook-routing\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task has been started from a library recipe yet — search the recipe library fo...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.10354300399816858`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 234969`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.8/10**, pass; unchanged. Advisory **notApplicable**. One start_work call correctly selects bundled bug-fix-tdd and dispatches the reproduction step. This scenario tests routing only; no repaired code or completed regression-test workflow is claimed. There are no declared artifact axes.\n\n\n\n## Positives\n\nThe request’s known root cause and test-first requirement map to the existing bug-fix-tdd recipe without creating a competing craftbook.\n\n## Negatives and diagnosis\n\nThe fixed routing success establishes correct workflow selection only. Seeded batches.ts still slices one element too far because the trial ends at dispatch. The8.8 quality component uses a stale pre-completion sniff, while the terminal result proves selection; fixed scores remain unchanged.\n\n## Performance and fit\n\nDecode 2.81tok/s, prefill 168.92tok/s; 347generated tokens. Peak GPU 10628/12227MiB, process RSS 9660MiB, whole-system RAM 26731/31865MiB. Context 65536, 1slot, f16; max observed released slot 12166. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nKeep the existing routing role concise: name the selected workflow and its first action without suggesting the actual repair has completed.\n\n## Strategic experiment — not applied\n\nReport routing, execution and deliverable verification as separate outcomes for short dispatch scenarios. Validate routing against unknown-cause incidents and known-cause repairs.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/facts.json): pass, fixed 8.8, 1.56minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/facts.json): pass, fixed 8.8, 1.28minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/facts.json): pass, fixed 8.8, 1.21minutes; other historical baseline; contextual only.\n\nEarlier same-scenario routing successes offer limited evidence of downstream code competence. The failed squisq repair and successful interface-contract show why dispatch must not be counted as end-to-end repair success.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nEvaluate recipe selection separately from the resulting task’s red-to-green evidence on a new broad baseline; retain this route-only result.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-interface-contract", "baselineCellId": "r1-saluki-cuda-developer-interface-contract", "suite": "developer", "scenario": "interface-contract", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T10:13:02.631Z", "finishedAt": "2026-11-10T10:27:19.763Z", "durationMs": 857132, "success": true, "reason": "tsc emit clean + node integration.mjs passed: 5 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 482318 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "interface-contract", "bytes": 1329, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.31475788857524034 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 6 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 482318 }, { "source": "facts", "path": "outcome.durationMs", "value": 857132 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 10, "band": "ship-ready" }, "facts": { "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "scenarioId": "interface-contract", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 141, "peakRssMb": 12034, "peakCpuPercent": 1228.2, "firstAtMs": 1, "lastAtMs": 838438 }, "gpu": { "available": true, "sampleCount": 141, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 141, "peakUsedMb": 28489, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 16787, "totalOutputTokens": 1943, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15400, "todayTokensCached": 14136, "todayTokensOut": 368, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15400, "totalTokensCached": 14136, "totalTokensOut": 368, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7776, "cachedInputTokens": 7068, "outputTokens": 253, "cost": 0, "durationMs": 98131, "outputTokensPerSec": 2.808221411408961, "at": "2026-11-10T10:19:18.859Z" }, "lastUpdated": "2026-11-10T10:19:18.859Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T10:27:09.173Z" } }, "source": "engine-log" }, "derived": { "durationMs": 840709, "meanTokensPerSec": 2.81, "genTokensPerSec": 2.81, "genTokens": 1943, "promptTokensPerSec": 152.31, "peakRssMb": 12034, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15400, "output": 368 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "event-pipeline/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-10T10:16:53.226Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "7354846c-45c7-48da-b427-944bb2a071f7" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T10:13:08.494Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T10:13:08.388Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "imara": 1, "jon": 2 }, "perTask": { "event-pipeline/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.1078033447265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1078033447265625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "tsc emit clean + node integration.mjs passed: 5 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "durationMs": 857132, "timeoutMs": 2723147, "budgetUsedFraction": 0.31475788857524034 }, "timing": { "startedAt": "2026-11-10T10:13:02.631Z", "finishedAt": "2026-11-10T10:27:19.763Z", "timeToFirstArtifactMs": 482318, "timeToLastArtifactWriteMs": 844685, "timeToFirstTokenMs": 63044, "firstTurnTtftMs": 51885, "timeToFirstToolCallMs": 230651 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Builder" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 6, "byTool": { "start_work": 1, "search_history": 1, "search_sessions": 1, "write_file": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 4 }, "sniff": { "progression": [ { "atMs": 846055, "filePath": "interface-contract", "bytes": 1329, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } ], "latest": { "filePath": "interface-contract", "bytes": 1329, "score": 5, "scoreMax": 5, "signals": [ "files-present", "tsc-clean", "integration-pass" ], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T10:13:02.635Z] [trial] id=interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj scenario=interface-contract model=underdog-saluki-27b-iq2-mix", "[2026-11-10T10:13:02.635Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "[2026-11-10T10:13:02.635Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T10:13:02.635Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-interface-contract-T8mesU", "[2026-11-10T10:13:02.635Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T10:13:02.638Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T10:13:02.641Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T10:13:02.648Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-interface-contract-T8mesU", "[2026-11-10T10:13:02.649Z] [trial] throughput-scaled ceiling: 30m → 45m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T10:13:08.368Z] [trial] daemon spawned pid=19524 port=https://127.0.0.1:53143", "[2026-11-10T10:13:09.376Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T10:13:09.394Z] [trial] meester=imara", "[2026-11-10T10:13:09.557Z] [trial] sent prompt to meester (1435 chars)", "[2026-11-10T10:13:09.616Z] [poll] initial digests hard=96801310840e3e3e soft=8c3851baf9a58108", "[2026-11-10T10:16:56.125Z] [poll] hard-progress: 96801310840e3e3e → bb080fb1b85edeb5 after 227s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=439,stream=810) sniff=0/0B)", "[2026-11-10T10:17:55.720Z] [poll] hard-progress: bb080fb1b85edeb5 → 0d6a89ba1ee5be8a after 60s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=1,slot=575,stream=942) sniff=0/0B)", "[2026-11-10T10:21:07.707Z] [poll] hard-progress: 0d6a89ba1ee5be8a → 08564e7950489dc8 after 192s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=937,stream=1378) sniff=1/0B)", "[2026-11-10T10:23:27.001Z] [poll] hard-progress: 08564e7950489dc8 → f97883822a65eeb4 after 139s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=2,tools=2,slot=1299,stream=1744) sniff=1/0B)", "[2026-11-10T10:23:39.016Z] [poll] hard-progress: f97883822a65eeb4 → 566e75e5dce5fe64 after 12s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=3,slot=1307,stream=1755) sniff=1/0B)", "[2026-11-10T10:25:25.896Z] [poll] hard-progress: 566e75e5dce5fe64 → 74c200b2e5915cf4 after 107s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=4,slot=1557,stream=2242) sniff=2/0B)", "[2026-11-10T10:26:07.811Z] [poll] hard-progress: 74c200b2e5915cf4 → 60e98bee3207029f after 42s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=3,tools=5,slot=1602,stream=2299) sniff=2/0B)", "[2026-11-10T10:27:08.692Z] [poll] terminal: success=true reason=tsc emit clean + node integration.mjs passed: 5 event(s) across 4 kind(s) routed and counted — team shape: 1 distinct non-meester session(s) (solo)", "[2026-11-10T10:27:09.217Z] [capture] dumped 3 session(s)", "[2026-11-10T10:27:09.220Z] [capture] snapshotted telemetry for 3 session(s)", "[2026-11-10T10:27:09.297Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)" ] }, "metrics": { "process": { "sampleCount": 141, "peakRssMb": 12034, "peakCpuPercent": 1228.2, "firstAtMs": 1, "lastAtMs": 838438 }, "gpu": { "available": true, "sampleCount": 141, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 141, "peakUsedMb": 28489, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 16787, "totalOutputTokens": 1943, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 2, "todayTokensIn": 15400, "todayTokensCached": 14136, "todayTokensOut": 368, "todayCost": 0, "totalTurns": 2, "totalTokensIn": 15400, "totalTokensCached": 14136, "totalTokensOut": 368, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 2 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 7776, "cachedInputTokens": 7068, "outputTokens": 253, "cost": 0, "durationMs": 98131, "outputTokensPerSec": 2.808221411408961, "at": "2026-11-10T10:19:18.859Z" }, "lastUpdated": "2026-11-10T10:19:18.859Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T10:27:09.173Z" } }, "source": "engine-log" }, "derived": { "durationMs": 840709, "meanTokensPerSec": 2.81, "genTokensPerSec": 2.81, "genTokens": 1943, "promptTokensPerSec": 152.31, "peakRssMb": 12034, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15400, "output": 368 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T10:13:02.631Z", "finishedAt": "2026-11-10T10:27:19.763Z", "timeToFirstArtifactMs": 482318, "timeToLastArtifactWriteMs": 844685, "timeToFirstTokenMs": 63044, "firstTurnTtftMs": 51885, "timeToFirstToolCallMs": 230651 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "event-pipeline/1", "stepId": "plan-and-execute", "activatedAt": "2026-11-10T10:16:53.226Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "7354846c-45c7-48da-b427-944bb2a071f7" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T10:13:08.494Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T10:13:08.388Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 3, "taskScoped": 1, "byGezel": { "imara": 1, "jon": 2 }, "perTask": { "event-pipeline/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 4, "maxContextFill": 0.1078033447265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1078033447265625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-interface-contract", "baselineCellId": "r1-saluki-cuda-developer-interface-contract", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "scenarioId": "interface-contract", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-10T17:01:36.804Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/types.ts", "artifactBytes": 315, "artifactSha256": "69418e415dfb47d45a850ca60d9a455e7bd1ba9bef822e4b0ec004989b5bf474", "axisSource": "evals/src/scenarios/interface-contract.ts; no declared axes", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "Three compact files share one type contract; the standard integration routes five events across four kinds and passes compilation. An advisory edge check finds plain-object count collisions for toString and __proto__, both allowed by kind:string. No subjective axes are declared.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/types.ts", "status": "notApplicable", "bytes": 315, "sha256": "69418e415dfb47d45a850ca60d9a455e7bd1ba9bef822e4b0ec004989b5bf474", "reviewedBytes": 315, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/producer.ts", "status": "notApplicable", "bytes": 607, "sha256": "a3b092a44698f23dd61016f2237b017c1911a0c80c29e7a04671c7cba2a32de7", "reviewedBytes": 607, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/consumer.ts", "status": "notApplicable", "bytes": 411, "sha256": "f66d051a37e824e8ce1f79da4e56d184826809999bc9564e3d028e8323175dc9", "reviewedBytes": 411, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T10:13:11.637Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T10:13:16.105Z INFO [chat] [llama-server] launch {\"pid\":9852,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T10:27:19.053Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-9852-1794306439053\",\"pid\":9852,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":842948,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 8601, "peakGpuMiB": 10628, "peakProcessRssMiB": 12034, "peakSystemRamMiB": 28489, "decodeTokensPerSec": 2.81, "prefillTokensPerSec": 152.31, "generatedTokens": 1943, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1078033447265625 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:23.921Z", "reviewId": "113", "cellId": "r3-saluki-cuda-developer-interface-contract", "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "scenarioId": "interface-contract", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "baselineCellId": "r1-saluki-cuda-developer-interface-contract", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "a7bfded13ec66a9238ac3952a0342135a802fab288584465280bceb8684d6779", "evidenceSha256": "62de620f386ed5d3e4be15f01981dbaec2dd3b2da53ab120be5b21a2d71e762f", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/types.ts", "bytes": 315, "sha256": "69418e415dfb47d45a850ca60d9a455e7bd1ba9bef822e4b0ec004989b5bf474", "shownBytes": 315, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/producer.ts", "bytes": 607, "sha256": "a3b092a44698f23dd61016f2237b017c1911a0c80c29e7a04671c7cba2a32de7", "shownBytes": 607, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/consumer.ts", "bytes": 411, "sha256": "f66d051a37e824e8ce1f79da4e56d184826809999bc9564e3d028e8323175dc9", "shownBytes": 411, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes are declared. Qualitatively, the three files match the spec. types.ts defines EventRecord and Sink. producer.ts returns 5 events of 4 kinds, each with a payload. consumer.ts calls sink.handle for every event and returns counts by kind. Both import from \"./types.ts\" with explicit extensions. The code looks type-correct.", "limitations": "Source-only review. tsc and integration.mjs were not run. The multi-developer split and the project creation are not visible in the evidence. The `import type` syntax may need allowImportingTsExtensions or a suitable tsconfig, which is unseen.", "meanScore": null, "durationMs": 14223, "costUsd": 0.014026, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 2678, "cache_read_input_tokens": 700, "output_tokens": 324, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 2678, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 324, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 2678, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2678 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 324, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 2678, "webSearchRequests": 0, "costUSD": 0.014026, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 2678, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - interface-contract / underdog-saluki-27b-iq2-mix\n\n**Composite: 10.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 14m 17s; budget used: 31.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 482318`; `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1329,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.31475788857524034`; `facts.toolUse.totalToolCalls: 6`; `facts.timing.timeToFirstArtifactMs: 482318`; `facts.outcome.durationMs: 857132` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 10 + 0.15 * 10 = **10.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12034 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28489 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.81 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 16787 + 1943 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 11% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 482318`\n- `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1329,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.31475788857524034`\n- `facts.toolUse.totalToolCalls: 6`\n- `facts.timing.timeToFirstArtifactMs: 482318`\n- `facts.outcome.durationMs: 857132`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 482318`; `facts.sniff.latest: {\"filePath\":\"interface-contract\",\"bytes\":1329,\"score\":5,\"scoreMax\":5,\"signals\":[\"files-present\",\"tsc-clean\",\"integration-pass\"],\"failReason\":null}`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.31475788857524034`; `facts.toolUse.totalToolCalls: 6`; `facts.timing.timeToFirstArtifactMs: 482318`; `facts.outcome.durationMs: 857132`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **10/10**, pass; unchanged. Advisory **notApplicable**. Three compact files share one type contract; the standard integration routes five events across four kinds and passes compilation. An advisory edge check finds plain-object count collisions for toString and __proto__, both allowed by kind:string. No subjective axes are declared.\n\n\n\n## Positives\n\nShared EventRecord/Sink types are imported consistently; producer and consumer are small and coherent. The existing runner confirms compilation and end-to-end routing/counting for ordinary event kinds. All payloads route to the supplied sink.\n\n## Negatives and diagnosis\n\ncounts starts as{}, so inherited keys are treated as existing counts. toString becomes a function-string plus1 and __proto__ is not an own numeric counter. This is a latent generic contract bug outside the standard fixture. One specialist session means success does not demonstrate inter-agent coordination.\n\n## Performance and fit\n\nDecode 2.81tok/s, prefill 152.31tok/s; 1943generated tokens. Peak GPU 10628/12227MiB, process RSS 12034MiB, whole-system RAM 28489/31865MiB. Context 65536, 1slot, f16; max observed released slot 8601. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nUse a null-prototype dictionary or Map for arbitrary string event kinds and check inherited names in the existing developer review. Do not apply the change to this frozen trial.\n\n## Strategic experiment — not applied\n\nSeparate shared-interface type compatibility from adversarial valid-key behavior. Test arbitrary identifiers across routing, state machines and data wrangling rather than adding one fixture exception.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/facts.json): pass, fixed 10, 5.81minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/facts.json): pass, fixed 10, 3.35minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/facts.json): pass, fixed 9.5, 0.88minutes; other historical baseline; contextual only.\n\nEarlier interface-contract trials also passed ordinary integration. The core failing-tests trial had a related inherited-property edge; this is a cross-scenario generalization weakness. Routing-only success is a narrower measurement.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nAdd a separate advisory identifier-boundary suite on a new baseline for event counters, state maps and object lookups, preserving all current fixed results.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-craftbook-code-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-code-review", "suite": "developer", "scenario": "craftbook-code-review", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T10:29:58.627Z", "finishedAt": "2026-11-10T10:57:21.922Z", "durationMs": 1643295, "success": false, "reason": "no real progress for 1094s — model busy but not delivering (hard digest stuck at 592144ad698e123f)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-code-review", "score": 1, "bytes": 0, "failReason": "reviews/rev-eval-1/report.md is 0 bytes, need ≥ 400", "deliverableMissing": true, "repairArtifactPath": "reviews/rev-eval-1/report.md", "milestones": 2 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1094s — model busy but not delivering (hard digest stuck at 592144ad698e123f)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-code-review", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, src/payment.js, docs/review-conventions.md" } } ] }, "quality": { "score": 0, "ruleId": "Q_NO_PRODUCED_OUTPUT", "summary": "No authored artifact event or scored sniff output was observed.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-code-review", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, src/payment.js, docs/review-conventions.md" } } ] }, "efficiency": { "score": 7.5, "ruleId": "E_ONE_MILD_INTERMEDIATE", "summary": "One efficiency dimension is efficient and the other is mildly intermediate.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.6034543856795098 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 4 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 1643295 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3, "band": "framework-gap" }, "facts": { "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "scenarioId": "craftbook-code-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 275, "peakRssMb": 11487, "peakCpuPercent": 1206.5, "firstAtMs": 1, "lastAtMs": 1630061 }, "gpu": { "available": true, "sampleCount": 275, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 275, "peakUsedMb": 29312, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 9660, "totalOutputTokens": 1264, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 9648, "todayTokensCached": 8929, "todayTokensOut": 77, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 9648, "totalTokensCached": 8929, "totalTokensOut": 77, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9648, "cachedInputTokens": 8929, "outputTokens": 77, "cost": 0, "durationMs": 523225, "outputTokensPerSec": 2.824823373272495, "at": "2026-11-10T10:38:56.921Z" }, "lastUpdated": "2026-11-10T10:38:56.921Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T10:57:16.397Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1631517, "meanTokensPerSec": 2.84, "genTokensPerSec": 2.84, "genTokens": 1264, "promptTokensPerSec": 159.19, "peakRssMb": 11487, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 9648, "output": 77 } } } }, "craftbook": { "craftbookId": "code-review", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "code-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T10:30:06.338Z", "completedAt": "2026-11-10T10:38:56.881Z", "ms": 530543, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "b7aa02ab-f0e8-4314-8903-912e4c2eca92" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-10T10:38:56.883Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T10:30:04.575Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T10:30:04.443Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 530543 }, "sessions": { "total": 1, "taskScoped": 1, "byGezel": { "daudi": 1 }, "perTask": { "code-review-workflow-eval/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.25, "maxMessages": 3, "maxContextFill": 0.1370086669921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1370086669921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1094s — model busy but not delivering (hard digest stuck at 592144ad698e123f)", "durationMs": 1643295, "timeoutMs": 2723147, "budgetUsedFraction": 0.6034543856795098 }, "timing": { "startedAt": "2026-11-10T10:29:58.627Z", "finishedAt": "2026-11-10T10:57:21.922Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 67331, "firstTurnTtftMs": 52262, "timeToFirstToolCallMs": 93097 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 4, "byTool": { "read_artifact": 2, "write_task_note": 1, "advance_task_step": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 14 }, "sniff": { "progression": [ { "atMs": 7959, "filePath": "craftbook-code-review", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, src/payment.js, docs/review-conventions.md" } ], "latest": { "filePath": "craftbook-code-review", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, src/payment.js, docs/review-conventions.md" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T10:29:58.631Z] [trial] id=craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb scenario=craftbook-code-review model=underdog-saluki-27b-iq2-mix", "[2026-11-10T10:29:58.631Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "[2026-11-10T10:29:58.631Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T10:29:58.632Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T10:29:58.631Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-code-review-t856t2", "[2026-11-10T10:29:58.635Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T10:29:58.639Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T10:29:58.646Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-code-review-t856t2", "[2026-11-10T10:29:58.646Z] [trial] throughput-scaled ceiling: 30m → 45m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T10:30:04.495Z] [trial] daemon spawned pid=10464 port=https://127.0.0.1:63921", "[2026-11-10T10:30:05.776Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T10:30:05.795Z] [trial] meester=elara", "[2026-11-10T10:30:05.795Z] [trial] running scenario setup hook", "[2026-11-10T10:30:06.480Z] [trial] scenario setup complete", "[2026-11-10T10:30:06.481Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T10:30:06.549Z] [poll] initial digests hard=2d4c9a2b40c89979 soft=6c5eff12e1474d3e", "[2026-11-10T10:30:06.648Z] [poll] hard-progress: 2d4c9a2b40c89979 → 9dafda407ef23eb5 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T10:30:11.847Z] [poll] hard-progress: 9dafda407ef23eb5 → ba7e8c749d28d410 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T10:31:35.891Z] [poll] hard-progress: ba7e8c749d28d410 → 61b0eee86a2355b2 after 84s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=63,stream=68) sniff=1/0B)", "[2026-11-10T10:31:59.690Z] [poll] hard-progress: 61b0eee86a2355b2 → 9e1c56e05d6fa5a8 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=95,stream=113) sniff=1/0B)", "[2026-11-10T10:38:26.204Z] [poll] hard-progress: 9e1c56e05d6fa5a8 → ca4efdede4050d4a after 387s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=1087,stream=1691) sniff=1/0B)", "[2026-11-10T10:39:02.240Z] [poll] hard-progress: ca4efdede4050d4a → 592144ad698e123f after 36s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1131,stream=1749) sniff=1/0B)", "[2026-11-10T10:57:15.947Z] [poll] no-progress (hard): no real progress for 1094s (threshold=1089s); model busy but not delivering — failing trial", "[2026-11-10T10:57:16.429Z] [capture] dumped 1 session(s)", "[2026-11-10T10:57:16.431Z] [capture] snapshotted telemetry for 1 session(s)" ] }, "metrics": { "process": { "sampleCount": 275, "peakRssMb": 11487, "peakCpuPercent": 1206.5, "firstAtMs": 1, "lastAtMs": 1630061 }, "gpu": { "available": true, "sampleCount": 275, "peakUtilPercent": 100, "peakMemUsedMb": 10626, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 275, "peakUsedMb": 29312, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 9660, "totalOutputTokens": 1264, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 9648, "todayTokensCached": 8929, "todayTokensOut": 77, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 9648, "totalTokensCached": 8929, "totalTokensOut": 77, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9648, "cachedInputTokens": 8929, "outputTokens": 77, "cost": 0, "durationMs": 523225, "outputTokensPerSec": 2.824823373272495, "at": "2026-11-10T10:38:56.921Z" }, "lastUpdated": "2026-11-10T10:38:56.921Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T10:57:16.397Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1631517, "meanTokensPerSec": 2.84, "genTokensPerSec": 2.84, "genTokens": 1264, "promptTokensPerSec": 159.19, "peakRssMb": 11487, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 9648, "output": 77 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T10:29:58.627Z", "finishedAt": "2026-11-10T10:57:21.922Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 67331, "firstTurnTtftMs": 52262, "timeToFirstToolCallMs": 93097 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "code-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T10:30:06.338Z", "completedAt": "2026-11-10T10:38:56.881Z", "ms": 530543, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "b7aa02ab-f0e8-4314-8903-912e4c2eca92" ] }, { "ref": "code-review-workflow-eval/1", "stepId": "report", "activatedAt": "2026-11-10T10:38:56.883Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T10:30:04.575Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T10:30:04.443Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 530543 }, "sessions": { "total": 1, "taskScoped": 1, "byGezel": { "daudi": 1 }, "perTask": { "code-review-workflow-eval/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.25, "maxMessages": 3, "maxContextFill": 0.1370086669921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1370086669921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-craftbook-code-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-code-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "scenarioId": "craftbook-code-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-10T17:01:38.665Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "artifactBytes": 0, "artifactSha256": null, "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/co/code-review/versions/1.0.4/test.json", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": {}, "meanScore": null, "justification": "The model reads the snapshot, records a sensible scope and advances to report, then produces no review artifact. It never reads the required local brief, payment source or conventions. No native/tool-bridge failure explains the stall; report-step write_artifact was available.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/artifacts/code-review-workflow-eval/reviews/rev-eval-1/changes.diff", "status": "notApplicable", "bytes": 4396, "sha256": "a1744bbef7b2c7cbd7ab3feba3df59a32a472f32781773715a67b8525758b728", "reviewedBytes": 4396, "truncated": false, "role": "seededInput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "status": "notApplicable", "bytes": 1113, "sha256": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "reviewedBytes": 1113, "truncated": false, "role": "seededInput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T10:30:11.668Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T10:30:16.930Z INFO [chat] [llama-server] launch {\"pid\":27076,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T10:57:21.207Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-27076-1794308241207\",\"pid\":27076,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":1624278,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 13038, "peakGpuMiB": 10626, "peakProcessRssMiB": 11487, "peakSystemRamMiB": 29312, "decodeTokensPerSec": 2.84, "prefillTokensPerSec": 159.19, "generatedTokens": 1264, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1370086669921875 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:27.979Z", "reviewId": "114", "cellId": "r3-saluki-cuda-developer-craftbook-code-review", "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "scenarioId": "craftbook-code-review", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "baselineCellId": "r1-saluki-cuda-developer-craftbook-code-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "4494cee3a7c98a2be409d450fd076cc698688ce4988abe2f82b46c756222b2e5", "evidenceSha256": "cf2cf7705d56ac6626b46eea8954e595b2102a479c34f758b68b061e392552e4", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json", "bytes": 1113, "sha256": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "shownBytes": 1113, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb\\workspace\\code-review-workflow-eval\\source\\craftbook-eval-brief.md", "bytes": 1372, "sha256": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "shownBytes": 1372, "truncated": false } ], "scoreAxes": {}, "justification": "The declared deliverable reviews/rev-eval-1/report.md is missing. Only the staged manifest.json is shown (7 files, commit review, diff 4396 chars). The task was at the scope step, and the report belongs to step 2. The evidence has no scope note, task notes, transcript or advance call, so the cause can't be determined.", "limitations": "No changes.diff, agent transcript, tool-call log or task notes were provided. I can't tell whether the run stopped early, failed at scope, or never reached the report step. Nothing here supports a score or a model success/failure verdict.", "meanScore": null, "durationMs": 14687, "costUsd": 0.016574, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3355, "cache_read_input_tokens": 700, "output_tokens": 308, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3355, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 308, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3355, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3355 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 308, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3355, "webSearchRequests": 0, "costUSD": 0.016574, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3355, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-code-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 3.0 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 27m 23s; budget used: 60.3%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at 592144ad698e123f)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md,...` |\n| Output quality (25%) | 0 / 10 | No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md,...` |\n| Process efficiency (20%) | 7.5 / 10 | One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.6034543856795098`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 1643295` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 0 + 0.2 * 7.5 + 0.15 * 10 = **3.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11487 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10626 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29312 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.84 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 9660 + 1264 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 4 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 531s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.25 | `continuity.sessions` |\n| Sessions reused across steps | 0 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 14% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at 592144ad698e123f)\"`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md,...`\n\n### Output quality - Q_NO_PRODUCED_OUTPUT\n\nNo authored artifact event or scored sniff output was observed.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md,...`\n\n### Process efficiency - E_ONE_MILD_INTERMEDIATE\n\nOne efficiency dimension is efficient and the other is mildly intermediate.\n\n- `facts.outcome.budgetUsedFraction: 0.6034543856795098`\n- `facts.toolUse.totalToolCalls: 4`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 1643295`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at 592144ad698e123f)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md,...`\n- Quality: No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-code-review\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md,...`\n- Efficiency: One efficiency dimension is efficient and the other is mildly intermediate. `facts.outcome.budgetUsedFraction: 0.6034543856795098`; `facts.toolUse.totalToolCalls: 4`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 1643295`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **3/10**, eligible task failure; unchanged. Advisory **noArtifact**. The model reads the snapshot, records a sensible scope and advances to report, then produces no review artifact. It never reads the required local brief, payment source or conventions. No native/tool-bridge failure explains the stall; report-step write_artifact was available.\n\n\n\n## Positives\n\nThe scope identifies genuine SQL construction, resource-close, swallowed-error and slice-bound concerns without mutating source. It preserves the manifest/diff and uses the intended first workflow transition.\n\n## Negatives and diagnosis\n\nThe report step never emits its declared artifact or verdict. Scoping themes are not a completed findings report, and missing conventions prevent confidently distinguishing deliberate null comparisons, caught background promises and Express arity from defects. Snapshot additions total84 rather than the scope note’s104.\n\n## Performance and fit\n\nDecode 2.84tok/s, prefill 159.19tok/s; 1264generated tokens. Peak GPU 10626/12227MiB, process RSS 11487MiB, whole-system RAM 29312/31865MiB. Context 65536, 1slot, f16; max observed released slot 13038. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nHave the existing reviewer read the required conventions and emit a concise findings table immediately after scoping, then refine evidence before the terminal verdict.\n\n## Strategic experiment — not applied\n\nMeasure step-handoff liveness and preserve a draft report before extended review. Validate that report-role write tools are available and distinguish no-output stalls from absent tool surfaces.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/facts.json): pass, fixed 10, 16.68minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/facts.json): pass, fixed 9.5, 26.56minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/facts.json): pass, fixed 9.5, 2.26minutes; other historical baseline; contextual only.\n\nEarlier same-scenario runs produced reports. This attempt resembles the squisq no-edit stall but reaches a real workflow handoff; it is not the Wikipedia missing-bridge issue. Interface-contract still completes in the same slow-throughput period.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nCompare bounded draft-first review on code-review, large-pr-review and API contracts; evaluate false positives and completion independently.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-craftbook-api-contract-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-api-contract-review", "suite": "developer", "scenario": "craftbook-api-contract-review", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T11:00:03.210Z", "finishedAt": "2026-11-10T11:54:05.937Z", "durationMs": 3242727, "success": false, "reason": "no real progress for 1093s — model busy but not delivering (hard digest stuck at d122259d9de64bdc)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-api-contract-review", "score": 3, "bytes": 0, "failReason": "tasks/eval/api-review.md is 0 bytes, need ≥ 1500", "deliverableMissing": true, "repairArtifactPath": "tasks/eval/api-review.md", "milestones": 2 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1093s — model busy but not delivering (hard digest stuck at d122259d9de64bdc)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1920737 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" } } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1920737 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 16 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1920737 }, { "source": "facts", "path": "outcome.durationMs", "value": 3242727 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4.5, "band": "framework-gap" }, "facts": { "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "scenarioId": "craftbook-api-contract-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 543, "peakRssMb": 13207, "peakCpuPercent": 1207, "firstAtMs": 1, "lastAtMs": 3228717 }, "gpu": { "available": true, "sampleCount": 543, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 543, "peakUsedMb": 29283, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 30818, "totalOutputTokens": 5202, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 15098, "todayTokensCached": 12919, "todayTokensOut": 1617, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 15098, "totalTokensCached": 12919, "totalTokensOut": 1617, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 15098, "cachedInputTokens": 12919, "outputTokens": 1617, "cost": 0, "durationMs": 1717870, "outputTokensPerSec": 2.6659101640669083, "at": "2026-11-10T11:32:03.946Z" }, "lastUpdated": "2026-11-10T11:32:03.946Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T11:53:59.771Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3230264, "meanTokensPerSec": 2.73, "genTokensPerSec": 2.73, "genTokens": 5202, "promptTokensPerSec": 163.65, "peakRssMb": 13207, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15098, "output": 1617 } } } }, "craftbook": { "craftbookId": "api-contract-review", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "api-contract-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T11:00:10.920Z", "completedAt": "2026-11-10T11:32:04.122Z", "ms": 1913202, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "18930173-431a-4b7d-99e2-ed21fb67f6dc" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "review", "activatedAt": "2026-11-10T11:32:04.123Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "9fd007c2-d1ed-4fdb-abea-9256a8fa2454" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T11:00:09.138Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T11:00:09.018Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1913202 }, "sessions": { "total": 2, "taskScoped": 2, "byGezel": { "kittipong": 1, "rayhane": 1 }, "perTask": { "api-contract-review-workflow-eval/1": 2 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.5, "maxMessages": 4, "maxContextFill": 0.1554107666015625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1554107666015625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1093s — model busy but not delivering (hard digest stuck at d122259d9de64bdc)", "durationMs": 3242727, "timeoutMs": 2723147, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-10T11:00:03.210Z", "finishedAt": "2026-11-10T11:54:05.937Z", "timeToFirstArtifactMs": 1920737, "timeToLastArtifactWriteMs": 1920737, "timeToFirstTokenMs": 72830, "firstTurnTtftMs": 57793, "timeToFirstToolCallMs": 200953 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 16, "byTool": { "read_file": 12, "list_dir": 1, "write_artifact": 2, "read_artifact": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 7961, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: source/craftbook-eval-brief.md, openapi.yaml, src/routes.ts, src/session-routes.ts, docs/api-contract.md" }, { "atMs": 203626, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 2, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" }, { "atMs": 1921462, "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" } ], "latest": { "filePath": "craftbook-api-contract-review", "bytes": 0, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task sourced from craftbook api-contract-review has not finished: its gates have not accepted the result yet" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T11:00:03.215Z] [trial] id=craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja scenario=craftbook-api-contract-review model=underdog-saluki-27b-iq2-mix", "[2026-11-10T11:00:03.215Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "[2026-11-10T11:00:03.215Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T11:00:03.215Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-api-contract-review-nskjvq", "[2026-11-10T11:00:03.215Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T11:00:03.218Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T11:00:03.222Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T11:00:03.230Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-api-contract-review-nskjvq", "[2026-11-10T11:00:03.231Z] [trial] throughput-scaled ceiling: 30m → 45m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T11:00:09.098Z] [trial] daemon spawned pid=14756 port=https://127.0.0.1:59255", "[2026-11-10T11:00:10.444Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T11:00:10.462Z] [trial] meester=fenton", "[2026-11-10T11:00:10.462Z] [trial] running scenario setup hook", "[2026-11-10T11:00:11.058Z] [trial] scenario setup complete", "[2026-11-10T11:00:11.060Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T11:00:11.119Z] [poll] initial digests hard=62e94f382a07c23d soft=4b3f07fe792a0a70", "[2026-11-10T11:00:11.238Z] [poll] hard-progress: 62e94f382a07c23d → 08540fdcb48e64dd after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T11:00:16.460Z] [poll] hard-progress: 08540fdcb48e64dd → 355ecee1aa901fce after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T11:03:26.889Z] [poll] hard-progress: 355ecee1aa901fce → 6f61f4946a020de5 after 190s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=6,slot=244,stream=285) sniff=2/0B)", "[2026-11-10T11:18:13.986Z] [poll] hard-progress: 6f61f4946a020de5 → 8385814f5da6177f after 887s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=7,slot=2391,stream=3915) sniff=2/0B)", "[2026-11-10T11:21:42.465Z] [poll] hard-progress: 8385814f5da6177f → 2041a5d8a4e6665f after 208s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=9,slot=2880,stream=4422) sniff=2/0B)", "[2026-11-10T11:32:04.735Z] [poll] hard-progress: 2041a5d8a4e6665f → 28a8eaf2252e40be after 622s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=2,tools=10,slot=4431,stream=7285) sniff=3/0B)", "[2026-11-10T11:32:10.631Z] [poll] hard-progress: 28a8eaf2252e40be → b307294ecc496ef7 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=3,tools=10,slot=4431,stream=7285) sniff=3/0B)", "[2026-11-10T11:35:46.163Z] [poll] hard-progress: b307294ecc496ef7 → d122259d9de64bdc after 216s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=3,tools=16,slot=4767,stream=7669) sniff=3/0B)", "[2026-11-10T11:45:35.031Z] [poll] hard ceiling reached but hard progress moved 589s ago — extending 15m (60m of 91m cap)" ] }, "metrics": { "process": { "sampleCount": 543, "peakRssMb": 13207, "peakCpuPercent": 1207, "firstAtMs": 1, "lastAtMs": 3228717 }, "gpu": { "available": true, "sampleCount": 543, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 543, "peakUsedMb": 29283, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 30818, "totalOutputTokens": 5202, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 15098, "todayTokensCached": 12919, "todayTokensOut": 1617, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 15098, "totalTokensCached": 12919, "totalTokensOut": 1617, "totalCost": 0, "medianOutputTokensPerSec": 2.7, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.7, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 15098, "cachedInputTokens": 12919, "outputTokens": 1617, "cost": 0, "durationMs": 1717870, "outputTokensPerSec": 2.6659101640669083, "at": "2026-11-10T11:32:03.946Z" }, "lastUpdated": "2026-11-10T11:32:03.946Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T11:53:59.771Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3230264, "meanTokensPerSec": 2.73, "genTokensPerSec": 2.73, "genTokens": 5202, "promptTokensPerSec": 163.65, "peakRssMb": 13207, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 15098, "output": 1617 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T11:00:03.210Z", "finishedAt": "2026-11-10T11:54:05.937Z", "timeToFirstArtifactMs": 1920737, "timeToLastArtifactWriteMs": 1920737, "timeToFirstTokenMs": 72830, "firstTurnTtftMs": 57793, "timeToFirstToolCallMs": 200953 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 4, "activationEvents": 1, "completed": 1, "gateApprovals": 1, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "api-contract-review-workflow-eval/1", "stepId": "scope", "activatedAt": "2026-11-10T11:00:10.920Z", "completedAt": "2026-11-10T11:32:04.122Z", "ms": 1913202, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "18930173-431a-4b7d-99e2-ed21fb67f6dc" ] }, { "ref": "api-contract-review-workflow-eval/1", "stepId": "review", "activatedAt": "2026-11-10T11:32:04.123Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "9fd007c2-d1ed-4fdb-abea-9256a8fa2454" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T11:00:09.138Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T11:00:09.018Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1913202 }, "sessions": { "total": 2, "taskScoped": 2, "byGezel": { "kittipong": 1, "rayhane": 1 }, "perTask": { "api-contract-review-workflow-eval/1": 2 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.5, "maxMessages": 4, "maxContextFill": 0.1554107666015625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1554107666015625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-craftbook-api-contract-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-api-contract-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "scenarioId": "craftbook-api-contract-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-10T17:03:24.411Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "artifactBytes": 0, "artifactSha256": null, "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/ap/api-contract-review/versions/1.1.5/test.json", "axes": [ { "name": "workflow fidelity", "description": "The result reflects the named craftbook sequence and its declared acceptance gates rather than a freehand substitute." }, { "name": "usefulness", "description": "The deliverable is coherent, actionable, and suitable for the craftbook purpose stated in the local brief." } ], "scoreAxes": {}, "meanScore": null, "justification": "Only scope.md exists; endpoint review, final report and verdict are absent. The scope inventories both route modules but overstates new-API status from version1.0.0. A grounding guard first rejects normative status-code guidance as unsupported facts; after a saved scope, repeated reads yield no review.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/artifacts/api-contract-review-workflow-eval/tasks/eval/review.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "status": "notApplicable", "bytes": 5345, "sha256": "6e1de595b2c87e7c4cb64701806f7671262abeef2edb3c351ee72571bc7253cb", "reviewedBytes": 5345, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/artifacts/api-contract-review-workflow-eval/tasks/eval/verdict.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T11:00:16.264Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T11:00:20.186Z INFO [chat] [llama-server] launch {\"pid\":12924,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T11:54:05.142Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-12924-1794311645142\",\"pid\":12924,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":3224957,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 16714, "peakGpuMiB": 10630, "peakProcessRssMiB": 13207, "peakSystemRamMiB": 29283, "decodeTokensPerSec": 2.73, "prefillTokensPerSec": 163.65, "generatedTokens": 5202, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1554107666015625 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:28.139Z", "reviewId": "115", "cellId": "r3-saluki-cuda-developer-craftbook-api-contract-review", "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "scenarioId": "craftbook-api-contract-review", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "baselineCellId": "r1-saluki-cuda-developer-craftbook-api-contract-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "01e81f2284b48f0536d10fb79e45b260283c5763d7bcd095ccd78ec300868aaa", "evidenceSha256": "dd4f7d95c2e3467a394f487be60a2ba853424c7b44c4b76dbc88366ec4f3c072", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md", "bytes": 5345, "sha256": "6e1de595b2c87e7c4cb64701806f7671262abeef2edb3c351ee72571bc7253cb", "shownBytes": 5345, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja\\workspace\\api-contract-review-workflow-eval\\source\\craftbook-eval-brief.md", "bytes": 2833, "sha256": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "shownBytes": 2833, "truncated": false } ], "scoreAxes": {}, "justification": "Only the scope step's scope.md (5,345 bytes, complete) exists. It maps the OpenAPI surface and locks an 8-item checklist. The declared deliverables api-review.md, review.md and verdict.md are missing, so there is no final report to score. The run stopped after scoping.", "limitations": "No evidence shows why the run stopped: no logs, gate results or advance_task_step call. Later steps may never have run, or their output may have been lost. scope.md is intermediate work, not a final deliverable. No success or failure is inferred, and no scores are given.", "meanScore": null, "durationMs": 14472, "costUsd": 0.026728, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5906, "cache_read_input_tokens": 700, "output_tokens": 303, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5906, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 303, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 5906, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5906 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 303, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 5906, "webSearchRequests": 0, "costUSD": 0.026728, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5906, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-api-contract-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 4.5 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 54m 3s; budget used: 100.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1093s — model busy but not delivering (hard digest stuck at d122259d9de64bdc)\"`; `facts.timing.timeToFirstArtifactMs: 1920737`; `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":0,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook api-contract-review has not finished: its gat...` |\n| Output quality (25%) | 5 / 10 | The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1920737`; `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":0,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook api-contract-review has not finished: its gat...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 16`; `facts.timing.timeToFirstArtifactMs: 1920737`; `facts.outcome.durationMs: 3242727` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 5 + 0.2 * 2.5 + 0.15 * 10 = **4.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13207 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29283 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.73 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 30818 + 5202 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 4 → 1 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 1913s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 2 / 0.50 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 16% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1093s — model busy but not delivering (hard digest stuck at d122259d9de64bdc)\"`\n- `facts.timing.timeToFirstArtifactMs: 1920737`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":0,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook api-contract-review has not finished: its gat...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 50% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 1920737`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":0,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook api-contract-review has not finished: its gat...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 16`\n- `facts.timing.timeToFirstArtifactMs: 1920737`\n- `facts.outcome.durationMs: 3242727`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1093s — model busy but not delivering (hard digest stuck at d122259d9de64bdc)\"`; `facts.timing.timeToFirstArtifactMs: 1920737`; `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":0,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook api-contract-review has not finished: its gat...`\n- Quality: The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1920737`; `facts.sniff.latest: {\"filePath\":\"craftbook-api-contract-review\",\"bytes\":0,\"score\":3,\"scoreMax\":6,\"signals\":[],\"failReason\":\"task sourced from craftbook api-contract-review has not finished: its gat...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 16`; `facts.timing.timeToFirstArtifactMs: 1920737`; `facts.outcome.durationMs: 3242727`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **4.5/10**, eligible task failure; unchanged. Advisory **noArtifact**. Only scope.md exists; endpoint review, final report and verdict are absent. The scope inventories both route modules but overstates new-API status from version1.0.0. A grounding guard first rejects normative status-code guidance as unsupported facts; after a saved scope, repeated reads yield no review.\n\n\n\n## Positives\n\nThe source, OpenAPI document and both route modules are read, including session registration and optional nullable-email conventions. The inventory correctly distinguishes documented history from the additional implemented widget lookup and records status retryCount as string in the spec.\n\n## Negatives and diagnosis\n\nNo actionable spec-anchored review is delivered. The initial write guard demands source evidence for proposed201/204 status-code criteria, blurring recommended policy with observed contract facts. The revised scope cites the local brief for those rules although it does not state those numeric codes. Declaring a NEW API solely from1.0.0/no previous local contract is unwarranted. Twelve source reads do not close the subsequent artifact step. No native or mock-bridge failure appears.\n\n## Performance and fit\n\nDecode 2.73tok/s, prefill 163.65tok/s; 5202generated tokens. Peak GPU 10630/12227MiB, process RSS 13207MiB, whole-system RAM 29283/31865MiB. Context 65536, 1slot, f16; max observed released slot 16714. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nHave the existing reviewer label recommendations as proposed checks and avoid inferring release history from a version string. Once scope is saved, emit the endpoint review against the original sources rather than rereading the entire set.\n\n## Strategic experiment — not applied\n\nGrounding validation should distinguish normative recommendations, quoted evidence and observed implementation facts. Test generic HTTP guidance and API lifecycle uncertainty across unrelated specs before changing the guard.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/facts.json): pass, fixed 9, 62.66minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/facts.json): pass, fixed 9, 55.16minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/facts.json): pass, fixed 8.5, 14.00minutes; other historical baseline; contextual only.\n\nEarlier same-scenario trials reached final reports. This attempt shares no-output handoff failure with code-review; its initial grounding rejection resembles the more consequential rejected systemic-analysis repair in deep-security-review.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nEvaluate an isolated grounding-guard test set of recommendations versus source claims, then a separate new-baseline API/code/security review sweep. Preserve this fixed eligible failure.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-craftbook-deep-security-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-deep-security-review", "suite": "developer", "scenario": "craftbook-deep-security-review", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T11:56:39.231Z", "finishedAt": "2026-11-10T12:47:31.212Z", "durationMs": 3051981, "success": false, "reason": "craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (2544 bytes) for 1002s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 10/11: security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 2544 bytes matches. Add that content.", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (2544 bytes) for 1002s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 10/11: security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 2544 bytes matches. Add that content." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 824440 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 2544, "score": 10, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 2544 bytes matches. Add that content." } } ] }, "quality": { "score": 9.1, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 90.9% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 824440 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-deep-security-review", "bytes": 2544, "score": 10, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 2544 bytes matches. Add that content." } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.7471701854717092 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 16 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 824440 }, { "source": "facts", "path": "outcome.durationMs", "value": 3051981 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 6, "band": "needs-tuning" }, "facts": { "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "scenarioId": "craftbook-deep-security-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 511, "peakRssMb": 13531, "peakCpuPercent": 1233, "firstAtMs": 1, "lastAtMs": 3038829 }, "gpu": { "available": true, "sampleCount": 511, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 511, "peakUsedMb": 29446, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 33766, "totalOutputTokens": 7266, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 8877, "todayTokensCached": 8777, "todayTokensOut": 185, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 8877, "totalTokensCached": 8777, "totalTokensOut": 185, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8877, "cachedInputTokens": 8777, "outputTokens": 185, "cost": 0, "durationMs": 2491304, "outputTokensPerSec": 2.8377665081090333, "at": "2026-11-10T12:43:20.133Z" }, "lastUpdated": "2026-11-10T12:43:20.133Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T12:47:25.714Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3040355, "meanTokensPerSec": 2.79, "genTokensPerSec": 2.79, "genTokens": 7266, "promptTokensPerSec": 157.7, "peakRssMb": 13531, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 8877, "output": 185 } } } }, "craftbook": { "craftbookId": "deep-security-review", "tasksObserved": 2, "craftbookTasks": 0, "reachedTerminal": false, "verdict": "artifact-only" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T11:56:45.214Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T11:56:45.111Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "rae": 1, "slobodan": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 3, "maxContextFill": 0.1104888916015625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1104888916015625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (2544 bytes) for 1002s — a re-emission of identical bytes counts as unchanged here after repair feedback and newer chat activity; still failing 10/11: security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 2544 bytes matches. Add that content.", "durationMs": 3051981, "timeoutMs": 4084720, "budgetUsedFraction": 0.7471701854717092 }, "timing": { "startedAt": "2026-11-10T11:56:39.231Z", "finishedAt": "2026-11-10T12:47:31.212Z", "timeToFirstArtifactMs": 824440, "timeToLastArtifactWriteMs": 2800903, "timeToFirstTokenMs": 71962, "firstTurnTtftMs": 60460, "timeToFirstToolCallMs": 119655 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 16, "byTool": { "read_file": 7, "list_scripts": 1, "write_file": 4, "read_files": 1, "stat": 2, "run_nodejs_script": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 11 }, "sniff": { "progression": [ { "atMs": 8000, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 0, "scoreMax": 11, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: src/admin.ts, src/profile.ts, src/reports.ts, docs/security.md" }, { "atMs": 121161, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 0, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 829755, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 3, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 1793008, "filePath": "craftbook-deep-security-review", "bytes": 0, "score": 4, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is 0 bytes, need ≥ 1500" }, { "atMs": 2043612, "filePath": "craftbook-deep-security-review", "bytes": 2544, "score": 10, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 2544 bytes matches. Add that content." } ], "latest": { "filePath": "craftbook-deep-security-review", "bytes": 2544, "score": 10, "scoreMax": 11, "signals": [], "failReason": "security-review/REPORT.md is missing required content /Systemic Themes|root cause|blast radius/ — nothing in its 2544 bytes matches. Add that content." } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T11:56:39.235Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "[2026-11-10T11:56:39.235Z] [trial] id=craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava scenario=craftbook-deep-security-review model=underdog-saluki-27b-iq2-mix", "[2026-11-10T11:56:39.236Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T11:56:39.236Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T11:56:39.235Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-deep-security-review-o7mAGF", "[2026-11-10T11:56:39.238Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T11:56:39.241Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T11:56:39.247Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-deep-security-review-o7mAGF", "[2026-11-10T11:56:39.247Z] [trial] throughput-scaled ceiling: 45m → 68m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T11:56:45.123Z] [trial] daemon spawned pid=52248 port=https://127.0.0.1:55622", "[2026-11-10T11:56:46.319Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T11:56:46.336Z] [trial] meester=linnea", "[2026-11-10T11:56:46.337Z] [trial] running scenario setup hook", "[2026-11-10T11:56:46.928Z] [trial] scenario setup complete", "[2026-11-10T11:56:46.929Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T11:56:47.026Z] [poll] initial digests hard=fdcd051ae53e39b2 soft=5fae3cbea4af7a5c", "[2026-11-10T11:56:47.381Z] [poll] hard-progress: fdcd051ae53e39b2 → cef97ace60a25207 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T11:58:40.460Z] [poll] hard-progress: cef97ace60a25207 → 8f9961c5257fb407 after 113s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=65,stream=85) sniff=0/0B)", "[2026-11-10T11:59:04.316Z] [poll] hard-progress: 8f9961c5257fb407 → 1d79cb6b1f25daf2 after 24s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=1,tools=4,slot=115,stream=135) sniff=0/0B)", "[2026-11-10T12:01:51.426Z] [poll] hard-progress: 1d79cb6b1f25daf2 → 158c423acc8a0edd after 167s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=541,stream=572) sniff=0/0B)", "[2026-11-10T12:03:03.123Z] [poll] hard-progress: 158c423acc8a0edd → edac697b030630d4 after 72s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=616,stream=702) sniff=0/0B)", "[2026-11-10T12:10:29.098Z] [poll] hard-progress: edac697b030630d4 → 8033dc3eb095ba35 after 446s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=1764,stream=2300) sniff=3/0B)", "[2026-11-10T12:26:32.345Z] [poll] hard-progress: 8033dc3eb095ba35 → b55e28c6c1205a95 after 963s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=10,slot=4148,stream=5066) sniff=4/0B)", "[2026-11-10T12:30:42.989Z] [poll] hard-progress: b55e28c6c1205a95 → 856dcdd4e5a94b91 after 251s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=11,slot=4811,stream=6372) sniff=10/2544B)", "[2026-11-10T12:40:31.529Z] [poll] hard-progress: 856dcdd4e5a94b91 → 1819c3d24f278fb3 after 589s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=12,slot=6350,stream=9308) sniff=10/2544B)" ] }, "metrics": { "process": { "sampleCount": 511, "peakRssMb": 13531, "peakCpuPercent": 1233, "firstAtMs": 1, "lastAtMs": 3038829 }, "gpu": { "available": true, "sampleCount": 511, "peakUtilPercent": 100, "peakMemUsedMb": 10630, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 511, "peakUsedMb": 29446, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 33766, "totalOutputTokens": 7266, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 1, "todayTokensIn": 8877, "todayTokensCached": 8777, "todayTokensOut": 185, "todayCost": 0, "totalTurns": 1, "totalTokensIn": 8877, "totalTokensCached": 8777, "totalTokensOut": 185, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 1 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8877, "cachedInputTokens": 8777, "outputTokens": 185, "cost": 0, "durationMs": 2491304, "outputTokensPerSec": 2.8377665081090333, "at": "2026-11-10T12:43:20.133Z" }, "lastUpdated": "2026-11-10T12:43:20.133Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T12:47:25.714Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3040355, "meanTokensPerSec": 2.79, "genTokensPerSec": 2.79, "genTokens": 7266, "promptTokensPerSec": 157.7, "peakRssMb": 13531, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 8877, "output": 185 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T11:56:39.231Z", "finishedAt": "2026-11-10T12:47:31.212Z", "timeToFirstArtifactMs": 824440, "timeToLastArtifactWriteMs": 2800903, "timeToFirstTokenMs": 71962, "firstTurnTtftMs": 60460, "timeToFirstToolCallMs": 119655 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T11:56:45.214Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T11:56:45.111Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 2, "taskScoped": 0, "byGezel": { "rae": 1, "slobodan": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 3, "maxContextFill": 0.1104888916015625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1104888916015625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-craftbook-deep-security-review", "baselineCellId": "r1-saluki-cuda-developer-craftbook-deep-security-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "scenarioId": "craftbook-deep-security-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T17:03:26.207Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/workspace/security-maintenance-eval/security-review/REPORT.md", "artifactBytes": 2556, "artifactSha256": "a0fd1fb60fe8925aa18fa5a363720d665180664621a955fc6a9fca71ca2c814b", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/de/deep-security-review/versions/1.1.5/test.json", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ], "scoreAxes": { "verification": 7, "systemic analysis": 2, "remediation": 7 }, "meanScore": 5.3, "justification": "Saved report/JSON identify all three seeded admin flaws and correctly exempt safe patterns; the recorded oracle passes. Saved prose lacks systemic analysis. A fuller repair containing Systemic Themes was rejected by a grounding guard despite the cited requireAdmin rule being present. Fixed stale-loop failure remains unchanged.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/workspace/security-maintenance-eval/security-review/REPORT.md", "status": "scored", "bytes": 2556, "sha256": "a0fd1fb60fe8925aa18fa5a363720d665180664621a955fc6a9fca71ca2c814b", "reviewedBytes": 2556, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/workspace/security-maintenance-eval/security-review/findings.json", "status": "notApplicable", "bytes": 1538, "sha256": "30f6f62327fb600940d6546ab05d94d355f7b71b5135db88c13774360a68366e", "reviewedBytes": 1538, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T11:56:48.817Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T11:56:53.092Z INFO [chat] [llama-server] launch {\"pid\":44668,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T12:47:30.506Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-44668-1794314850506\",\"pid\":44668,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":3037414,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 16593, "peakGpuMiB": 10630, "peakProcessRssMiB": 13531, "peakSystemRamMiB": 29446, "decodeTokensPerSec": 2.79, "prefillTokensPerSec": 157.7, "generatedTokens": 7266, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1104888916015625 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:40.547Z", "reviewId": "116", "cellId": "r3-saluki-cuda-developer-craftbook-deep-security-review", "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "scenarioId": "craftbook-deep-security-review", "status": "scored", "axes": [ { "name": "verification", "description": "Raw leads are confirmed as real source-to-sink or authorization paths." }, { "name": "systemic analysis", "description": "Findings are grouped by root cause and blast radius." }, { "name": "remediation", "description": "The report and JSON contain concrete, severity-consistent fixes." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "baselineCellId": "r1-saluki-cuda-developer-craftbook-deep-security-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "bf84c347a201e7863d42367060f79b5b97c58165f290e875f260eab1b347d2e6", "evidenceSha256": "2b92ad0fdc6e3bbf3733412493b4f157f11b22d3b94e406b781ee6d161cd69c3", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/workspace/security-maintenance-eval/security-review/REPORT.md", "bytes": 2556, "sha256": "a0fd1fb60fe8925aa18fa5a363720d665180664621a955fc6a9fca71ca2c814b", "shownBytes": 2556, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/workspace/security-maintenance-eval/security-review/findings.json", "bytes": 1538, "sha256": "30f6f62327fb600940d6546ab05d94d355f7b71b5135db88c13774360a68366e", "shownBytes": 1538, "truncated": false } ], "contextFiles": [], "scoreAxes": { "verification": 7, "systemic analysis": 4, "remediation": 7 }, "justification": "Three findings are tied to code snippets and sinks. Decoys in reports.ts and profile.ts (fixed exec, allow-listed ORDER BY, /health) are explicitly cleared. Fixes are concrete (execFile, parameterized query, requireAdmin) and severities are consistent. Weaknesses: no root-cause grouping or blast-radius analysis, per-finding themes only, and CWE-666 is wrong for missing authorization.", "limitations": "Source files were not supplied, so the line numbers and the non-finding claims could not be checked against the code. The scores rely on the report's own quoted snippets and on its description of docs/security.md.", "meanScore": 6, "durationMs": 16525, "costUsd": 0.0211908, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3729, "cache_read_input_tokens": 808, "output_tokens": 619, "output_tokens_details": { "thinking_tokens": 265 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3729, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 619, "cache_read_input_tokens": 808, "cache_creation_input_tokens": 3729, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3729 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 619, "cacheReadInputTokens": 808, "cacheCreationInputTokens": 3729, "webSearchRequests": 0, "costUSD": 0.0211908, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 265, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3729, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-deep-security-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 6.0 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 50m 52s; budget used: 74.7%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (2544 bytes) for 1002s — a re-emission of identical bytes counts as u...`; `facts.timing.timeToFirstArtifactMs: 824440`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":2544,\"score\":10,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content /Systemic The...` |\n| Output quality (25%) | 9.1 / 10 | The latest scenario gate reached 90.9% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 824440`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":2544,\"score\":10,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content /Systemic The...` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.7471701854717092`; `facts.toolUse.totalToolCalls: 16`; `facts.timing.timeToFirstArtifactMs: 824440`; `facts.outcome.durationMs: 3051981` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 9.1 + 0.2 * 5 + 0.15 * 10 = **6.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 13531 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10630 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 29446 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.79 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 33766 + 7266 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 11% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (2544 bytes) for 1002s — a re-emission of identical bytes counts as u...`\n- `facts.timing.timeToFirstArtifactMs: 824440`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":2544,\"score\":10,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content /Systemic The...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 90.9% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 824440`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":2544,\"score\":10,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content /Systemic The...`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.7471701854717092`\n- `facts.toolUse.totalToolCalls: 16`\n- `facts.timing.timeToFirstArtifactMs: 824440`\n- `facts.outcome.durationMs: 3051981`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"craftbook-deep-security-review stale no-write repair loop: security-review/REPORT.md had unchanged content (2544 bytes) for 1002s — a re-emission of identical bytes counts as u...`; `facts.timing.timeToFirstArtifactMs: 824440`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":2544,\"score\":10,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content /Systemic The...`\n- Quality: The latest scenario gate reached 90.9% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 824440`; `facts.sniff.latest: {\"filePath\":\"craftbook-deep-security-review\",\"bytes\":2544,\"score\":10,\"scoreMax\":11,\"signals\":[],\"failReason\":\"security-review/REPORT.md is missing required content /Systemic The...`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.7471701854717092`; `facts.toolUse.totalToolCalls: 16`; `facts.timing.timeToFirstArtifactMs: 824440`; `facts.outcome.durationMs: 3051981`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **6/10**, eligible task failure; unchanged. Advisory **5.3/10**. Saved report/JSON identify all three seeded admin flaws and correctly exempt safe patterns; the recorded oracle passes. Saved prose lacks systemic analysis. A fuller repair containing Systemic Themes was rejected by a grounding guard despite the cited requireAdmin rule being present. Fixed stale-loop failure remains unchanged.\n\nExact declared axes: verification=7; systemic analysis=2; remediation=7.\n\n## Positives\n\nActual source-to-shell, source-to-SQL and missing-authorization paths are identified at the right lines with concrete parameterization/middleware fixes. The local oracle confirms3/3 seeded defects and zero false positives on safe files. The model reads the security conventions and does not report the deliberate exceptions.\n\n## Negatives and diagnosis\n\nThe saved report never gains root-cause/blast-radius grouping, so the exact declared axis remains weak. Crucially, the attempted replacement DOES contain a substantial Systemic Themes section, but write_file rejects a sentence quoting docs/security.md. The source explicitly says All /admin routes must require the requireAdmin middleware before business logic runs; the guard complaint strips the inline-code identifier. This is a likely grounding false positive contributing to the stale repair outcome. The model rereads but never saves a corrected repair. Rejected text is evidence of an attempt, not the scored deliverable.\n\n## Performance and fit\n\nDecode 2.79tok/s, prefill 157.7tok/s; 7266generated tokens. Peak GPU 10630/12227MiB, process RSS 13531MiB, whole-system RAM 29446/31865MiB. Context 65536, 1slot, f16; max observed released slot 16593. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing reviewer role, respond to a rejected write by citing the precise source sentence and making a targeted supported repair. Prefer non-shell execution plus validated paths over claiming an allowlist alone proves all safety properties.\n\n## Strategic experiment — not applied\n\nTest grounding normalization of inline-code tokens and quotation matching. Distinguish substantive missing analysis from blocked publication of an attempted repair in diagnostics, while leaving fixed outcome rules intact.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/facts.json): pass, fixed 9, 33.43minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/facts.json): fail, fixed 5.3, 19.52minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/facts.json): fail, fixed 6.3, 3.20minutes; other historical baseline; contextual only.\n\nEarlier same-scenario trials wrote longer systemic reports. This attempt’s detection precision is good despite fixed failure; its lost repair is a framework contribution distinct from missing-bridge Wikipedia acquisition and no-output code-review stalls.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nBuild a separate guard regression set for code identifiers inside quotations, then assess a new multi-scenario review baseline. Do not rescore, retry or publish the rejected repair here.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-large-pr-review", "baselineCellId": "r1-saluki-cuda-developer-large-pr-review", "suite": "developer", "scenario": "large-pr-review", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T12:50:20.835Z", "finishedAt": "2026-11-10T13:14:42.675Z", "durationMs": 1461840, "success": false, "reason": "no real progress for 1091s — model busy but not delivering (hard digest stuck at e8bde910b20416a9)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "large-pr-review", "score": 0, "bytes": 0, "repairFilePath": "pr-review.md", "deliverableMissing": true, "failReason": "pr-review.md does not exist yet" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1091s — model busy but not delivering (hard digest stuck at e8bde910b20416a9)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "quality": { "score": 0, "ruleId": "Q_NO_PRODUCED_OUTPUT", "summary": "No authored artifact event or scored sniff output was observed.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": null } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.26841009104724994 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 5 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 1461840 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.5, "band": "framework-gap" }, "facts": { "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "scenarioId": "large-pr-review", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 243, "peakRssMb": 9746, "peakCpuPercent": 185060.6, "firstAtMs": 0, "lastAtMs": 1444673 }, "gpu": { "available": true, "sampleCount": 243, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 243, "peakUsedMb": 26814, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 23581, "totalOutputTokens": 399, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T13:14:33.320Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1446118, "meanTokensPerSec": 2.65, "genTokensPerSec": 2.65, "genTokens": 399, "promptTokensPerSec": 130.07, "peakRssMb": 9746, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T12:50:26.923Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T12:50:26.769Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "rina": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1758880615234375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1758880615234375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 2, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1091s — model busy but not delivering (hard digest stuck at e8bde910b20416a9)", "durationMs": 1461840, "timeoutMs": 5446293, "budgetUsedFraction": 0.26841009104724994 }, "timing": { "startedAt": "2026-11-10T12:50:20.835Z", "finishedAt": "2026-11-10T13:14:42.675Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 106747, "firstTurnTtftMs": 94000, "timeToFirstToolCallMs": 140301 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 5, "byTool": { "list_artifacts": 1, "list_dir": 1, "read_artifact": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 127 }, "sniff": { "progression": [], "latest": null }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T12:50:20.840Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "[2026-11-10T12:50:20.840Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T12:50:20.840Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T12:50:20.839Z] [trial] id=large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29 scenario=large-pr-review model=underdog-saluki-27b-iq2-mix", "[2026-11-10T12:50:20.840Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-large-pr-review-S3iGFW", "[2026-11-10T12:50:20.844Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T12:50:20.847Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T12:50:20.854Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-large-pr-review-S3iGFW", "[2026-11-10T12:50:20.855Z] [trial] throughput-scaled ceiling: 60m → 91m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T12:50:26.855Z] [trial] daemon spawned pid=51880 port=https://127.0.0.1:55845", "[2026-11-10T12:50:28.128Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T12:50:28.150Z] [trial] meester=soren", "[2026-11-10T12:50:28.150Z] [trial] running scenario setup hook", "[2026-11-10T12:50:29.739Z] [trial] scenario setup complete", "[2026-11-10T12:50:29.740Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T12:50:29.823Z] [poll] initial digests hard=d803162db5a13bdb soft=3bb0f4716a6855a3", "[2026-11-10T12:50:29.972Z] [poll] hard-progress: d803162db5a13bdb → cc6e2e78513c58ec after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=0/0B)", "[2026-11-10T12:52:41.958Z] [poll] hard-progress: cc6e2e78513c58ec → d57bf6ca92e29d40 after 132s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=55,stream=65) sniff=0/0B)", "[2026-11-10T12:56:21.781Z] [poll] hard-progress: d57bf6ca92e29d40 → e8bde910b20416a9 after 220s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=296,stream=369) sniff=0/0B)", "[2026-11-10T13:14:32.880Z] [poll] no-progress (hard): no real progress for 1091s (threshold=1089s); model busy but not delivering — failing trial", "[2026-11-10T13:14:33.356Z] [capture] dumped 1 session(s)", "[2026-11-10T13:14:33.359Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-10T13:14:33.359Z] [capture] INCOMPLETE TRANSCRIPT for 1 session(s) — rina: 0 persisted vs 5 recorded. Assistant turns commit at turn END, so a mid-turn termination loses them. Triage from session-telemetry.json and daemon.log, NOT from the empty transcript.", "[2026-11-10T13:14:33.718Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-10T13:14:42.673Z] [trial] daemon shut down" ] }, "metrics": { "process": { "sampleCount": 243, "peakRssMb": 9746, "peakCpuPercent": 185060.6, "firstAtMs": 0, "lastAtMs": 1444673 }, "gpu": { "available": true, "sampleCount": 243, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 243, "peakUsedMb": 26814, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 23581, "totalOutputTokens": 399, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T13:14:33.320Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1446118, "meanTokensPerSec": 2.65, "genTokensPerSec": 2.65, "genTokens": 399, "promptTokensPerSec": 130.07, "peakRssMb": 9746, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T12:50:20.835Z", "finishedAt": "2026-11-10T13:14:42.675Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 106747, "firstTurnTtftMs": 94000, "timeToFirstToolCallMs": 140301 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 2 }, "steps": { "activated": 2, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T12:50:26.923Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T12:50:26.769Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "rina": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.1758880615234375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1758880615234375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-large-pr-review", "baselineCellId": "r1-saluki-cuda-developer-large-pr-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "scenarioId": "large-pr-review", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-10T17:03:27.813Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/workspace/large-pr-review-eval/pr-review.md", "artifactBytes": 0, "artifactSha256": null, "axisSource": "evals/src/scenarios/large-pr-review.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No review or coverage ledger was produced. Five calls list the workspace/artifacts and read the PR overview,120-file manifest and flags, but no individual diff/source review follows. The watchdog stops the trial; write and review tools were present and no mock bridge failure is recorded.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/workspace/large-pr-review-eval/pr-review.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/workspace/large-pr-review-eval/pr-review-coverage.json", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T12:50:31.656Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T12:50:37.062Z INFO [chat] [llama-server] launch {\"pid\":40876,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T13:14:41.969Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-40876-1794316481969\",\"pid\":40876,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":1444908,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 41836, "peakGpuMiB": 10640, "peakProcessRssMiB": 9746, "peakSystemRamMiB": 26814, "decodeTokensPerSec": 2.65, "prefillTokensPerSec": 130.07, "generatedTokens": 399, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1758880615234375 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:42.280Z", "reviewId": "117", "cellId": "r3-saluki-cuda-developer-large-pr-review", "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "scenarioId": "large-pr-review", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "baselineCellId": "r1-saluki-cuda-developer-large-pr-review", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "420b3baecca73f8980a91f876bdc272cf16091e08eb1317358afa3531dbb0f60", "evidenceSha256": "560fdd306e37c0a27c3080cd15fd6dee9b778c9baf813e43041b5cae09d40a55", "artifactFiles": [], "contextFiles": [], "scoreAxes": {}, "justification": "Both declared deliverables, pr-review.md and pr-review-coverage.json, were absent at closeout. The artifacts list is empty and no context files or transcript were supplied. The runner ended terminal, so there is no evidence of batch progress, coverage, findings, or the authorization defect. No scores are given.", "limitations": "Only the prompt and missing-path metadata were available. The cause of the absence (interruption, early stop, wrong path, or write failure) can't be determined. No partial work was observed, so this is neither a model failure nor a success. Nothing was executed or retried.", "meanScore": null, "durationMs": 14224, "costUsd": 0.009507999999999999, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1591, "cache_read_input_tokens": 700, "output_tokens": 307, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1591, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 307, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1591, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1591 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 307, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1591, "webSearchRequests": 0, "costUSD": 0.009507999999999999, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1591, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - large-pr-review / underdog-saluki-27b-iq2-mix\n\n**Composite: 3.5 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 24m 22s; budget used: 26.8%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1091s — model busy but not delivering (hard digest stuck at e8bde910b20416a9)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null` |\n| Output quality (25%) | 0 / 10 | No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.26841009104724994`; `facts.toolUse.totalToolCalls: 5`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 1461840` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 0 + 0.2 * 10 + 0.15 * 10 = **3.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9746 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26814 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.65 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 23581 + 399 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 2s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 2 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 18% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 2 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1091s — model busy but not delivering (hard digest stuck at e8bde910b20416a9)\"`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: null`\n\n### Output quality - Q_NO_PRODUCED_OUTPUT\n\nNo authored artifact event or scored sniff output was observed.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: null`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.26841009104724994`\n- `facts.toolUse.totalToolCalls: 5`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 1461840`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1091s — model busy but not delivering (hard digest stuck at e8bde910b20416a9)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null`\n- Quality: No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: null`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.26841009104724994`; `facts.toolUse.totalToolCalls: 5`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 1461840`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **3.5/10**, eligible task failure; unchanged. Advisory **noArtifact**. No review or coverage ledger was produced. Five calls list the workspace/artifacts and read the PR overview,120-file manifest and flags, but no individual diff/source review follows. The watchdog stops the trial; write and review tools were present and no mock bridge failure is recorded.\n\n\n\n## Positives\n\nThe model discovers the full PR inventory and reads the overview/flags rather than assuming the first page is the whole change.\n\n## Negatives and diagnosis\n\nReading the inventory is not reviewing120files. Neither the declared report nor coverage artifact exists, and the three local source files receive no read calls. The late authorization defect and assured-API negative control are therefore not evaluated. This is a no-output capability outcome, not an artifact-quality score.\n\n## Performance and fit\n\nDecode 2.65tok/s, prefill 130.07tok/s; 399generated tokens. Peak GPU 10640/12227MiB, process RSS 9746MiB, whole-system RAM 26814/31865MiB. Context 65536, 1slot, f16; max observed released slot 41836. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nUse the existing reviewer role to create an explicit pending/reviewed ledger from the manifest and review bounded batches, persisting findings after each batch. Never mark inventory-only files as reviewed.\n\n## Strategic experiment — not applied\n\nExpose resumable bounded review batches with evidence-backed coverage, measuring first substantive finding and coverage progress independently. Validate on different PR sizes and mixed safe/unsafe changes.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/facts.json): pass, fixed 8.5, 101.64minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/facts.json): pass, fixed 8.5, 86.70minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/facts.json): pass, fixed 8.5, 23.00minutes; other historical baseline; contextual only.\n\nEarlier same-scenario trials produced reports and coverage; this trial does not. It echoes code-review and API review stalls during the same slow decode period, while interface-contract still completes, so throughput is a limitation rather than a proven sole cause.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nCompare a general incremental coverage workflow on large PRs, security sweeps and API inventories in a fresh baseline with unchanged quality criteria.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-craftbook-codemod-sweep", "baselineCellId": "r1-saluki-cuda-developer-craftbook-codemod-sweep", "suite": "developer", "scenario": "craftbook-codemod-sweep", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T13:17:14.804Z", "finishedAt": "2026-11-10T14:25:22.625Z", "durationMs": 4087821, "success": false, "reason": "no real progress for 1094s — model busy but not delivering (hard digest stuck at d256c0b4719b9122)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-codemod-sweep", "score": 27, "bytes": 1812, "failReason": "tasks/1/review.md is 0 bytes, need ≥ 400", "deliverableMissing": true, "repairArtifactPath": "tasks/1/review.md", "milestones": 4 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1094s — model busy but not delivering (hard digest stuck at d256c0b4719b9122)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1293677 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 1812, "score": 27, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" } } ] }, "quality": { "score": 8.4, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 84.4% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1293677 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-codemod-sweep", "bytes": 1812, "score": 27, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" } } ] }, "efficiency": { "score": 5, "ruleId": "E_MODERATE", "summary": "The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.5003795853816173 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 29 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1293677 }, { "source": "facts", "path": "outcome.durationMs", "value": 4087821 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.8, "band": "needs-tuning" }, "facts": { "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "scenarioId": "craftbook-codemod-sweep", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 684, "peakRssMb": 15314, "peakCpuPercent": 1207.2, "firstAtMs": 0, "lastAtMs": 4073029 }, "gpu": { "available": true, "sampleCount": 684, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 684, "peakUsedMb": 31242, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 42034, "totalOutputTokens": 6921, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 32669, "todayTokensCached": 31149, "todayTokensOut": 2295, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 32669, "totalTokensCached": 31149, "totalTokensOut": 2295, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11142, "cachedInputTokens": 10395, "outputTokens": 823, "cost": 0, "durationMs": 558014, "outputTokensPerSec": 2.831761338327614, "at": "2026-11-10T13:59:55.081Z" }, "lastUpdated": "2026-11-10T13:59:55.081Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T14:25:15.790Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4075050, "meanTokensPerSec": 2.83, "genTokensPerSec": 2.83, "genTokens": 6921, "promptTokensPerSec": 157.34, "peakRssMb": 15314, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 32669, "output": 2295 } } } }, "craftbook": { "craftbookId": "codemod-sweep", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 6, "activationEvents": 3, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "profile-cards/1", "stepId": "enumerate", "activatedAt": "2026-11-10T13:17:22.241Z", "completedAt": "2026-11-10T13:38:48.671Z", "ms": 1286430, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "911b31eb-1648-4f67-9ba2-c9256a9fb437" ] }, { "ref": "profile-cards/1", "stepId": "apply", "activatedAt": "2026-11-10T13:38:48.672Z", "completedAt": "2026-11-10T13:49:32.487Z", "ms": 643815, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "fcef9ad6-0a6d-4a49-bd0a-29e5b3cd3976" ] }, { "ref": "profile-cards/1", "stepId": "verify", "activatedAt": "2026-11-10T13:49:32.488Z", "completedAt": "2026-11-10T13:59:55.311Z", "ms": 622823, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "fcef9ad6-0a6d-4a49-bd0a-29e5b3cd3976" ] }, { "ref": "profile-cards/1", "stepId": "evaluate", "activatedAt": "2026-11-10T13:59:55.313Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "e58e6c78-4f46-4349-8e24-2c6797a13cbf" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T13:17:20.783Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T13:17:20.653Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 643815 }, "sessions": { "total": 5, "taskScoped": 3, "byGezel": { "barno": 1, "dawit": 1, "erwan": 1, "fenton": 1, "shivam": 1 }, "perTask": { "profile-cards/1": 3 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5, "maxMessages": 4, "maxContextFill": 0.1287841796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1287841796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1094s — model busy but not delivering (hard digest stuck at d256c0b4719b9122)", "durationMs": 4087821, "timeoutMs": 8169440, "budgetUsedFraction": 0.5003795853816173 }, "timing": { "startedAt": "2026-11-10T13:17:14.804Z", "finishedAt": "2026-11-10T14:25:22.625Z", "timeToFirstArtifactMs": 1293677, "timeToLastArtifactWriteMs": 2560278, "timeToFirstTokenMs": 74678, "firstTurnTtftMs": 59631, "timeToFirstToolCallMs": 126450 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Codebase Analyst", "Developer", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 29, "byTool": { "read_file": 6, "grep_files": 1, "list_dir": 1, "write_artifact": 4, "read_artifact": 1, "read_files": 4, "run_package_script": 3, "replace_in_file": 7, "grep_artifact": 1, "read_artifacts": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 15 }, "sniff": { "progression": [ { "atMs": 8136, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 9, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/api.js, src/index.js, src/profile.js, tests/profile.test.mjs, CHANGELOG.md, source/rename-brief.md" }, { "atMs": 127225, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 9, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/api.js, src/index.js, src/profile.js, tests/profile.test.mjs, CHANGELOG.md" }, { "atMs": 342268, "filePath": "craftbook-codemod-sweep", "bytes": 0, "score": 9, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json" }, { "atMs": 1297341, "filePath": "craftbook-codemod-sweep", "bytes": 1812, "score": 14, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json" }, { "atMs": 1735846, "filePath": "craftbook-codemod-sweep", "bytes": 1812, "score": 15, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json" }, { "atMs": 1741834, "filePath": "craftbook-codemod-sweep", "bytes": 1812, "score": 19, "scoreMax": 32, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json" }, { "atMs": 1938825, "filePath": "craftbook-codemod-sweep", "bytes": 1812, "score": 24, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" }, { "atMs": 2560489, "filePath": "craftbook-codemod-sweep", "bytes": 1812, "score": 27, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" } ], "latest": { "filePath": "craftbook-codemod-sweep", "bytes": 1812, "score": 27, "scoreMax": 32, "signals": [], "failReason": "task sourced from craftbook codemod-sweep has not finished: its gates have not accepted the result yet" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T13:17:14.808Z] [trial] id=craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3 scenario=craftbook-codemod-sweep model=underdog-saluki-27b-iq2-mix", "[2026-11-10T13:17:14.808Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-codemod-sweep-nxU4nS", "[2026-11-10T13:17:14.808Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "[2026-11-10T13:17:14.809Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T13:17:14.812Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T13:17:14.809Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T13:17:14.816Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T13:17:14.823Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-codemod-sweep-nxU4nS", "[2026-11-10T13:17:14.823Z] [trial] throughput-scaled ceiling: 90m → 136m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T13:17:20.698Z] [trial] daemon spawned pid=28336 port=https://127.0.0.1:56373", "[2026-11-10T13:17:21.754Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-10T13:17:21.772Z] [trial] meester=fenton", "[2026-11-10T13:17:21.773Z] [trial] running scenario setup hook", "[2026-11-10T13:17:22.399Z] [trial] scenario setup complete", "[2026-11-10T13:17:22.400Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T13:17:22.470Z] [poll] initial digests hard=bdbea91aa1ed67af soft=f61e4cbf39f8fed8", "[2026-11-10T13:17:22.996Z] [poll] hard-progress: bdbea91aa1ed67af → 8387843748ca2a7a after 1s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-10T13:17:28.558Z] [poll] hard-progress: 8387843748ca2a7a → 9d47359c6227df10 after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-10T13:19:22.074Z] [poll] hard-progress: 9d47359c6227df10 → 890b746c90b58b43 after 114s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=83,stream=97) sniff=9/0B)", "[2026-11-10T13:22:57.145Z] [poll] hard-progress: 890b746c90b58b43 → d2b29a40fbcb7e23 after 215s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=548,stream=598) sniff=9/0B)", "[2026-11-10T13:32:56.602Z] [poll] hard-progress: d2b29a40fbcb7e23 → 0ccad67a27a13a0e after 599s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=2126,stream=2581) sniff=9/0B)", "[2026-11-10T13:38:52.196Z] [poll] hard-progress: 0ccad67a27a13a0e → 06d30243063a0919 after 356s (workspaceProjects=3 sessions=3 daemon[src=service-telemetry](turns=1,tools=10,slot=3059,stream=4023) sniff=14/1812B)", "[2026-11-10T13:39:54.431Z] [poll] hard-progress: 06d30243063a0919 → ab76c3b615b4a4b5 after 62s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=2,tools=10,slot=3059,stream=4023) sniff=14/1812B)", "[2026-11-10T13:41:12.459Z] [poll] hard-progress: ab76c3b615b4a4b5 → 941d54e890c7a426 after 78s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=3,tools=11,slot=3092,stream=4057) sniff=14/1812B)", "[2026-11-10T13:41:54.516Z] [poll] hard-progress: 941d54e890c7a426 → f6f9b5a0391ee67a after 42s (workspaceProjects=3 sessions=4 daemon[src=service-telemetry](turns=3,tools=12,slot=3173,stream=4176) sniff=14/1812B)" ] }, "metrics": { "process": { "sampleCount": 684, "peakRssMb": 15314, "peakCpuPercent": 1207.2, "firstAtMs": 0, "lastAtMs": 4073029 }, "gpu": { "available": true, "sampleCount": 684, "peakUtilPercent": 100, "peakMemUsedMb": 10638, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 684, "peakUsedMb": 31242, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 42034, "totalOutputTokens": 6921, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 3, "todayTokensIn": 32669, "todayTokensCached": 31149, "todayTokensOut": 2295, "todayCost": 0, "totalTurns": 3, "totalTokensIn": 32669, "totalTokensCached": 31149, "totalTokensOut": 2295, "totalCost": 0, "medianOutputTokensPerSec": 2.8, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.8, "turns": 3 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11142, "cachedInputTokens": 10395, "outputTokens": 823, "cost": 0, "durationMs": 558014, "outputTokensPerSec": 2.831761338327614, "at": "2026-11-10T13:59:55.081Z" }, "lastUpdated": "2026-11-10T13:59:55.081Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T14:25:15.790Z" } }, "source": "engine-log" }, "derived": { "durationMs": 4075050, "meanTokensPerSec": 2.83, "genTokensPerSec": 2.83, "genTokens": 6921, "promptTokensPerSec": 157.34, "peakRssMb": 15314, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 32669, "output": 2295 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T13:17:14.804Z", "finishedAt": "2026-11-10T14:25:22.625Z", "timeToFirstArtifactMs": 1293677, "timeToLastArtifactWriteMs": 2560278, "timeToFirstTokenMs": 74678, "firstTurnTtftMs": 59631, "timeToFirstToolCallMs": 126450 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 6, "activationEvents": 3, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "profile-cards/1", "stepId": "enumerate", "activatedAt": "2026-11-10T13:17:22.241Z", "completedAt": "2026-11-10T13:38:48.671Z", "ms": 1286430, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "911b31eb-1648-4f67-9ba2-c9256a9fb437" ] }, { "ref": "profile-cards/1", "stepId": "apply", "activatedAt": "2026-11-10T13:38:48.672Z", "completedAt": "2026-11-10T13:49:32.487Z", "ms": 643815, "gateRejections": 0, "toolCalls": 12, "sessionIds": [ "fcef9ad6-0a6d-4a49-bd0a-29e5b3cd3976" ] }, { "ref": "profile-cards/1", "stepId": "verify", "activatedAt": "2026-11-10T13:49:32.488Z", "completedAt": "2026-11-10T13:59:55.311Z", "ms": 622823, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "fcef9ad6-0a6d-4a49-bd0a-29e5b3cd3976" ] }, { "ref": "profile-cards/1", "stepId": "evaluate", "activatedAt": "2026-11-10T13:59:55.313Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "e58e6c78-4f46-4349-8e24-2c6797a13cbf" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T13:17:20.783Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T13:17:20.653Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 643815 }, "sessions": { "total": 5, "taskScoped": 3, "byGezel": { "barno": 1, "dawit": 1, "erwan": 1, "fenton": 1, "shivam": 1 }, "perTask": { "profile-cards/1": 3 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5, "maxMessages": 4, "maxContextFill": 0.1287841796875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1287841796875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-craftbook-codemod-sweep", "baselineCellId": "r1-saluki-cuda-developer-craftbook-codemod-sweep", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "scenarioId": "craftbook-codemod-sweep", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T17:05:58.517Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/verification.md", "artifactBytes": 2551, "artifactSha256": "e2fd8b7557cb66911e7f6ccf31866fe5fc4cbf1dbd55dd6b55190a43764e5771", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/co/codemod-sweep/versions/1.0.6/test.json", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ], "scoreAxes": { "Enumeration completeness": 8, "Invariant discipline": 8, "Reconciled verification": 3 }, "meanScore": 6.3, "justification": "The initial search enumerates11sites; seven precise replacements preserve behavior and three real suite runs pass. Verification falsely says a residual workspace search was rerun and reports11 residual matches while listing four. No review artifact closes the workflow, so the fixed result remains a stall.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/verification.md", "status": "scored", "bytes": 2551, "sha256": "e2fd8b7557cb66911e7f6ccf31866fe5fc4cbf1dbd55dd6b55190a43764e5771", "reviewedBytes": 2551, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/sites.md", "status": "notApplicable", "bytes": 1842, "sha256": "b8f1af8b1d652462da4c310953ddb5545156f7d816fc2240ef79c15afba8b6d3", "reviewedBytes": 1842, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/change-notes.md", "status": "notApplicable", "bytes": 1459, "sha256": "7775292e85e003cc5f9aaae107200ceb2eb5cd18d459cc789ff8c23679222dab", "reviewedBytes": 1459, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/review.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/api.js", "status": "notApplicable", "bytes": 225, "sha256": "bd78e33b9cf74aec788b859016f4a43e161b0173bd06add93ecc140b791c7a0d", "reviewedBytes": 225, "truncated": false, "role": "modifiedSource" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/index.js", "status": "notApplicable", "bytes": 37, "sha256": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c", "reviewedBytes": 37, "truncated": false, "role": "modifiedSource" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/profile.js", "status": "notApplicable", "bytes": 229, "sha256": "fb079d5420e1e811a50449f8b905369da1359f6bd1d800053b350e323b763284", "reviewedBytes": 229, "truncated": false, "role": "modifiedSource" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/tests/profile.test.mjs", "status": "notApplicable", "bytes": 558, "sha256": "1f687e56bf6fd264f98007c0fef446bf87f248ab076394785ae5145f3501f9a7", "reviewedBytes": 558, "truncated": false, "role": "modifiedSource" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T13:17:27.885Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T13:17:31.685Z INFO [chat] [llama-server] launch {\"pid\":44312,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T14:25:21.872Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-44312-1794320721872\",\"pid\":44312,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":4070187,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 13723, "peakGpuMiB": 10638, "peakProcessRssMiB": 15314, "peakSystemRamMiB": 31242, "decodeTokensPerSec": 2.83, "prefillTokensPerSec": 157.34, "generatedTokens": 6921, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1287841796875 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:46.869Z", "reviewId": "118", "cellId": "r3-saluki-cuda-developer-craftbook-codemod-sweep", "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "scenarioId": "craftbook-codemod-sweep", "status": "scored", "axes": [ { "name": "Enumeration completeness", "description": "Every site was found by real searches before any edit - the definition, the re-export, the imports and call sites, and the log string - with the changelog excluded for a stated reason rather than missed." }, { "name": "Invariant discipline", "description": "The change stayed mechanical: nothing changed beyond the declared rename, the excluded files are untouched, and behavior is identical." }, { "name": "Reconciled verification", "description": "The residual re-scan is quoted with its result, the found/changed/excluded counts add up exactly, and the suite run backing the green claim is real." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "baselineCellId": "r1-saluki-cuda-developer-craftbook-codemod-sweep", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "99b9d53633893a564f78da7a903df1356239880c9f5617c5691245085cd4940f", "evidenceSha256": "ff2f76e287c4680708c5bafb71cec29c8b887a809e3793187e84b981497727dc", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/verification.md", "bytes": 2551, "sha256": "e2fd8b7557cb66911e7f6ccf31866fe5fc4cbf1dbd55dd6b55190a43764e5771", "shownBytes": 2551, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/sites.md", "bytes": 1842, "sha256": "b8f1af8b1d652462da4c310953ddb5545156f7d816fc2240ef79c15afba8b6d3", "shownBytes": 1842, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/change-notes.md", "bytes": 1459, "sha256": "7775292e85e003cc5f9aaae107200ceb2eb5cd18d459cc789ff8c23679222dab", "shownBytes": 1459, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/api.js", "bytes": 225, "sha256": "bd78e33b9cf74aec788b859016f4a43e161b0173bd06add93ecc140b791c7a0d", "shownBytes": 225, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/index.js", "bytes": 37, "sha256": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c", "shownBytes": 37, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/profile.js", "bytes": 229, "sha256": "fb079d5420e1e811a50449f8b905369da1359f6bd1d800053b350e323b763284", "shownBytes": 229, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/tests/profile.test.mjs", "bytes": 558, "sha256": "1f687e56bf6fd264f98007c0fef446bf87f248ab076394785ae5145f3501f9a7", "shownBytes": 558, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3\\workspace\\profile-cards\\source\\rename-brief.md", "bytes": 534, "sha256": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "shownBytes": 534, "truncated": false } ], "scoreAxes": { "Enumeration completeness": 8, "Invariant discipline": 8, "Reconciled verification": 5 }, "justification": "sites.md lists all 11 sites (definition, re-export, imports, calls, log string) with file:line, and excludes the changelog and brief with reasons. Edits are a pure rename and the exclusions still hold the old name. verification.md says \"11 matches\" for the residual scan but lists 4, and its table has a \"wait, that's 6\" slip.", "limitations": "I can't confirm the quoted suite output or the claimed before-and-after runs. review.md is missing, though it belongs to a later step. The scan was not independently re-run. The changelog file contents beyond two quoted lines were not shown.", "meanScore": 7, "durationMs": 18631, "costUsd": 0.034665, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 6572, "cache_read_input_tokens": 830, "output_tokens": 829, "output_tokens_details": { "thinking_tokens": 486 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 6572, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 829, "cache_read_input_tokens": 830, "cache_creation_input_tokens": 6572, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6572 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 829, "cacheReadInputTokens": 830, "cacheCreationInputTokens": 6572, "webSearchRequests": 0, "costUSD": 0.034665, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 486, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 6572, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-codemod-sweep / underdog-saluki-27b-iq2-mix\n\n**Composite: 5.8 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 68m 8s; budget used: 50.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at d256c0b4719b9122)\"`; `facts.timing.timeToFirstArtifactMs: 1293677`; `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1812,\"score\":27,\"scoreMax\":32,\"signals\":[],\"failReason\":\"task sourced from craftbook codemod-sweep has not finished: its gates have...` |\n| Output quality (25%) | 8.4 / 10 | The latest scenario gate reached 84.4% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1293677`; `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1812,\"score\":27,\"scoreMax\":32,\"signals\":[],\"failReason\":\"task sourced from craftbook codemod-sweep has not finished: its gates have...` |\n| Process efficiency (20%) | 5 / 10 | The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.5003795853816173`; `facts.toolUse.totalToolCalls: 29`; `facts.timing.timeToFirstArtifactMs: 1293677`; `facts.outcome.durationMs: 4087821` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 8.4 + 0.2 * 5 + 0.15 * 10 = **5.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15314 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10638 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31242 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.83 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 42034 + 6921 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 6 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 644s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 3 / 0.50 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 13% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at d256c0b4719b9122)\"`\n- `facts.timing.timeToFirstArtifactMs: 1293677`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1812,\"score\":27,\"scoreMax\":32,\"signals\":[],\"failReason\":\"task sourced from craftbook codemod-sweep has not finished: its gates have...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 84.4% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 1293677`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1812,\"score\":27,\"scoreMax\":32,\"signals\":[],\"failReason\":\"task sourced from craftbook codemod-sweep has not finished: its gates have...`\n\n### Process efficiency - E_MODERATE\n\nThe non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions.\n\n- `facts.outcome.budgetUsedFraction: 0.5003795853816173`\n- `facts.toolUse.totalToolCalls: 29`\n- `facts.timing.timeToFirstArtifactMs: 1293677`\n- `facts.outcome.durationMs: 4087821`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at d256c0b4719b9122)\"`; `facts.timing.timeToFirstArtifactMs: 1293677`; `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1812,\"score\":27,\"scoreMax\":32,\"signals\":[],\"failReason\":\"task sourced from craftbook codemod-sweep has not finished: its gates have...`\n- Quality: The latest scenario gate reached 84.4% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 1293677`; `facts.sniff.latest: {\"filePath\":\"craftbook-codemod-sweep\",\"bytes\":1812,\"score\":27,\"scoreMax\":32,\"signals\":[],\"failReason\":\"task sourced from craftbook codemod-sweep has not finished: its gates have...`\n- Efficiency: The non-severe efficiency evidence is moderate, including a 20-29 call trace or two intermediate dimensions. `facts.outcome.budgetUsedFraction: 0.5003795853816173`; `facts.toolUse.totalToolCalls: 29`; `facts.timing.timeToFirstArtifactMs: 1293677`; `facts.outcome.durationMs: 4087821`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **5.8/10**, eligible task failure; unchanged. Advisory **6.3/10**. The initial search enumerates11sites; seven precise replacements preserve behavior and three real suite runs pass. Verification falsely says a residual workspace search was rerun and reports11 residual matches while listing four. No review artifact closes the workflow, so the fixed result remains a stall.\n\nExact declared axes: Enumeration completeness=8; Invariant discipline=8; Reconciled verification=3.\n\n## Positives\n\nDefinition, re-export, imports, calls and log string all become loadUser; changelog and task brief retain the old spelling intentionally. Real baseline/post-edit/verification suite receipts each show3/3pass. The mechanical code change is correct on the inspected files.\n\n## Negatives and diagnosis\n\nOnly one grep_files call occurred before edits; later grep_artifact searches change-notes, not the workspace. Thus the claimed repeated whole-tree scan has no receipt. Final code reads support the absence of old names in known code sites but not a performed scan. The11 residual claim contradicts four excluded occurrences and the7+4 reconciliation. Review.md is absent. An earlier guard rejected a standalone derived count11 despite search evidence, adding friction, but a revised sites document was saved.\n\n## Performance and fit\n\nDecode 2.83tok/s, prefill 157.34tok/s; 6921generated tokens. Peak GPU 10638/12227MiB, process RSS 15314MiB, whole-system RAM 31242/31865MiB. Context 65536, 1slot, f16; max observed released slot 13723. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing verifier role, run the exact residual workspace search and quote its actual result; distinguish found-before, changed and remaining counts. Persist the final review after reconciling arithmetic.\n\n## Strategic experiment — not applied\n\nAttach search/test receipt IDs to verification claims and derive occurrence totals deterministically. Test on renames involving comments, logs, re-exports and legitimate exclusions across unrelated projects.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/facts.json): pass, fixed 8.5, 31.93minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/facts.json): pass, fixed 8.5, 28.09minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/facts.json): fail, fixed 5.3, 16.80minutes; other historical baseline; contextual only.\n\nEarlier same-scenario trials closed the workflow. This code change succeeds mechanically while the verification/reporting stage fails, unlike refactor-module where no restructuring occurs. TDD records real red/green evidence more faithfully.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nCompare evidence-linked verification on codemod, refactor and schema migration in a fresh baseline; do not retry this capability failure.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-craftbook-refactor-module", "baselineCellId": "r1-saluki-cuda-developer-craftbook-refactor-module", "suite": "developer", "scenario": "craftbook-refactor-module", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T14:28:09.483Z", "finishedAt": "2026-11-10T15:17:28.526Z", "durationMs": 2959043, "success": false, "reason": "no real progress for 1094s — model busy but not delivering (hard digest stuck at 3bfa0133424494ed)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-refactor-module", "score": 6, "bytes": 0, "failReason": "tasks/1/baseline.md is 0 bytes, need ≥ 500", "deliverableMissing": true, "repairArtifactPath": "tasks/1/baseline.md", "milestones": 1 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 1094s — model busy but not delivering (hard digest stuck at 3bfa0133424494ed)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 529301 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 0, "score": 6, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" } } ] }, "quality": { "score": 2, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 20% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 529301 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-refactor-module", "bytes": 0, "score": 6, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.27165658626366723 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 9 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 529301 }, { "source": "facts", "path": "outcome.durationMs", "value": 2959043 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 4, "band": "framework-gap" }, "facts": { "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "scenarioId": "craftbook-refactor-module", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 494, "peakRssMb": 11102, "peakCpuPercent": 1213.7, "firstAtMs": 1, "lastAtMs": 2942817 }, "gpu": { "available": true, "sampleCount": 494, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 494, "peakUsedMb": 27990, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 13558, "totalOutputTokens": 4746, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T15:17:20.315Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2944571, "meanTokensPerSec": 2.76, "genTokensPerSec": 2.76, "genTokens": 4746, "promptTokensPerSec": 147.26, "peakRssMb": 11102, "peakGpuUtilPercent": 100, "billing": null } }, "craftbook": { "craftbookId": "refactor-module", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "crewlog/1", "stepId": "baseline", "activatedAt": "2026-11-10T14:28:17.172Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "5ab18f34-8997-4657-a89a-1f8e3e21e0c4" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T14:28:15.423Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T14:28:15.299Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 1, "byGezel": { "dilara": 1 }, "perTask": { "crewlog/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 1, "maxContextFill": 0.1375732421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1375732421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 1094s — model busy but not delivering (hard digest stuck at 3bfa0133424494ed)", "durationMs": 2959043, "timeoutMs": 10892587, "budgetUsedFraction": 0.27165658626366723 }, "timing": { "startedAt": "2026-11-10T14:28:09.483Z", "finishedAt": "2026-11-10T15:17:28.526Z", "timeToFirstArtifactMs": 529301, "timeToLastArtifactWriteMs": 1852508, "timeToFirstTokenMs": 80969, "firstTurnTtftMs": 66053, "timeToFirstToolCallMs": 125395 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 9, "byTool": { "read_files": 1, "run_package_script": 2, "write_file": 2, "write_artifact": 2, "replace_in_file": 1, "copy_artifact_to_workspace": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 14 }, "sniff": { "progression": [ { "atMs": 8338, "filePath": "craftbook-refactor-module", "bytes": 0, "score": 5, "scoreMax": 30, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/format-name.js, src/report.js, tests/format.test.mjs, tests/report.test.mjs, source/refactor-brief.md" }, { "atMs": 126493, "filePath": "craftbook-refactor-module", "bytes": 0, "score": 6, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" } ], "latest": { "filePath": "craftbook-refactor-module", "bytes": 0, "score": 6, "scoreMax": 30, "signals": [], "failReason": "task sourced from craftbook refactor-module has not finished: its gates have not accepted the result yet" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T14:28:09.493Z] [trial] id=craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4 scenario=craftbook-refactor-module model=underdog-saluki-27b-iq2-mix", "[2026-11-10T14:28:09.493Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T14:28:09.493Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "[2026-11-10T14:28:09.493Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-refactor-module-0JcVrw", "[2026-11-10T14:28:09.496Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T14:28:09.493Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T14:28:09.500Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T14:28:09.507Z] [trial] throughput-scaled ceiling: 120m → 182m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T14:28:09.507Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-refactor-module-0JcVrw", "[2026-11-10T14:28:15.264Z] [trial] daemon spawned pid=29580 port=https://127.0.0.1:61746", "[2026-11-10T14:28:16.724Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-10T14:28:16.740Z] [trial] meester=ines", "[2026-11-10T14:28:16.740Z] [trial] running scenario setup hook", "[2026-11-10T14:28:17.377Z] [trial] scenario setup complete", "[2026-11-10T14:28:17.379Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T14:28:17.445Z] [poll] initial digests hard=656a80eca3ee36aa soft=97d93274fabb49f3", "[2026-11-10T14:28:17.874Z] [poll] hard-progress: 656a80eca3ee36aa → 914d6cdcc47499eb after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-10T14:28:23.211Z] [poll] hard-progress: 914d6cdcc47499eb → a724404b8aa371aa after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-10T14:30:16.052Z] [poll] hard-progress: a724404b8aa371aa → 8954bfb137efd61f after 113s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=113,stream=163) sniff=6/0B)", "[2026-11-10T14:30:40.159Z] [poll] hard-progress: 8954bfb137efd61f → 3b7255577c49d83b after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=133,stream=187) sniff=6/0B)", "[2026-11-10T14:37:02.294Z] [poll] hard-progress: 3b7255577c49d83b → ac4f03ebd2adb17d after 382s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=1108,stream=1637) sniff=6/0B)", "[2026-11-10T14:37:26.056Z] [poll] hard-progress: ac4f03ebd2adb17d → 14cb77c92f1ac807 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1133,stream=1666) sniff=6/0B)", "[2026-11-10T14:45:35.286Z] [poll] hard-progress: 14cb77c92f1ac807 → af70660adc4edc93 after 489s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=2369,stream=3610) sniff=6/0B)", "[2026-11-10T14:46:40.335Z] [poll] hard-progress: af70660adc4edc93 → beae5529e5fe6931 after 65s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=2485,stream=3813) sniff=6/0B)", "[2026-11-10T14:51:34.676Z] [poll] hard-progress: beae5529e5fe6931 → 02dbac18ef2667ad after 294s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=3197,stream=5222) sniff=6/0B)" ] }, "metrics": { "process": { "sampleCount": 494, "peakRssMb": 11102, "peakCpuPercent": 1213.7, "firstAtMs": 1, "lastAtMs": 2942817 }, "gpu": { "available": true, "sampleCount": 494, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 494, "peakUsedMb": 27990, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 13558, "totalOutputTokens": 4746, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T15:17:20.315Z" } }, "source": "engine-log" }, "derived": { "durationMs": 2944571, "meanTokensPerSec": 2.76, "genTokensPerSec": 2.76, "genTokens": 4746, "promptTokensPerSec": 147.26, "peakRssMb": 11102, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T14:28:09.483Z", "finishedAt": "2026-11-10T15:17:28.526Z", "timeToFirstArtifactMs": 529301, "timeToLastArtifactWriteMs": 1852508, "timeToFirstTokenMs": 80969, "firstTurnTtftMs": 66053, "timeToFirstToolCallMs": 125395 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "crewlog/1", "stepId": "baseline", "activatedAt": "2026-11-10T14:28:17.172Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "5ab18f34-8997-4657-a89a-1f8e3e21e0c4" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T14:28:15.423Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T14:28:15.299Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 1, "byGezel": { "dilara": 1 }, "perTask": { "crewlog/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.3333333333333333, "maxMessages": 1, "maxContextFill": 0.1375732421875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1375732421875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-craftbook-refactor-module", "baselineCellId": "r1-saluki-cuda-developer-craftbook-refactor-module", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "scenarioId": "craftbook-refactor-module", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-10T17:06:00.458Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/change-notes.md", "artifactBytes": 0, "artifactSha256": null, "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/re/refactor-module/versions/2.0.7/test.json", "axes": [ { "name": "Guardrail-first discipline", "description": "The suite was genuinely run and recorded green before any restructuring, characterization coverage was considered honestly, and the baseline counted the duplication with a stated method." }, { "name": "Reversible-stage execution", "description": "The restructure happened in small planned stages at the real sites, with the suite kept green between stages and deviations recorded honestly rather than papered over." }, { "name": "Genuine deduplication", "description": "One shared implementation remains, both call sites use it, public exports survived unchanged, and no behavior change was smuggled in under cover of the refactor." } ], "scoreAxes": {}, "meanScore": null, "justification": "No refactor or declared baseline/plan/change-notes/verification/review was delivered. The model runs2green tests, adds11useful characterization tests and gets13green, then spends the remaining work on unexecuted measurement helpers. Original duplicate implementations remain; normalize-name.js is absent.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/change-notes.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/baseline.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/plan.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/verification.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/review.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/measure.mjs", "status": "notApplicable", "bytes": 2437, "sha256": "7f2ddb4fae250e215a0ae9878837b653a19eac3cbcee06127750feb72fc5e470", "reviewedBytes": 2437, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/.measure-baseline.mjs", "status": "notApplicable", "bytes": 2437, "sha256": "7f2ddb4fae250e215a0ae9878837b653a19eac3cbcee06127750feb72fc5e470", "reviewedBytes": 2437, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/tests/characterization.test.mjs", "status": "notApplicable", "bytes": 1751, "sha256": "04a801e3542a014a4f26b3b4c34904dbcf8760938c2b54bf4b9a0cee82087d19", "reviewedBytes": 1751, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/tests/measure.test.mjs", "status": "notApplicable", "bytes": 2607, "sha256": "212d1175e046b301f92e13295c83ef68e27263b3a9134833b019abaea58fe16e", "reviewedBytes": 2607, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/src/normalize-name.js", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T14:28:22.537Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T14:28:26.277Z INFO [chat] [llama-server] launch {\"pid\":30056,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T15:17:27.732Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-30056-1794323847732\",\"pid\":30056,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":2941456,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 16557, "peakGpuMiB": 10628, "peakProcessRssMiB": 11102, "peakSystemRamMiB": 27990, "decodeTokensPerSec": 2.76, "prefillTokensPerSec": 147.26, "generatedTokens": 4746, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1375732421875 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:55.424Z", "reviewId": "119", "cellId": "r3-saluki-cuda-developer-craftbook-refactor-module", "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "scenarioId": "craftbook-refactor-module", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "baselineCellId": "r1-saluki-cuda-developer-craftbook-refactor-module", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "42f5ed45b166320ff0ac45e0c0da24b9505a1689c5b35d84d3b1a2f3e9c057e5", "evidenceSha256": "9818ba8a9f38500c03ff44d3cc081de45752da5f9abbc121fbae2af5bbad6b6a", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/measure.mjs", "bytes": 2437, "sha256": "7f2ddb4fae250e215a0ae9878837b653a19eac3cbcee06127750feb72fc5e470", "shownBytes": 2437, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/.measure-baseline.mjs", "bytes": 2437, "sha256": "7f2ddb4fae250e215a0ae9878837b653a19eac3cbcee06127750feb72fc5e470", "shownBytes": 2437, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/tests/characterization.test.mjs", "bytes": 1751, "sha256": "04a801e3542a014a4f26b3b4c34904dbcf8760938c2b54bf4b9a0cee82087d19", "shownBytes": 1751, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/tests/measure.test.mjs", "bytes": 2607, "sha256": "212d1175e046b301f92e13295c83ef68e27263b3a9134833b019abaea58fe16e", "shownBytes": 2607, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4\\workspace\\crewlog\\source\\refactor-brief.md", "bytes": 720, "sha256": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "shownBytes": 720, "truncated": false } ], "scoreAxes": {}, "justification": "The baseline step's primary deliverable, artifacts/crewlog/tasks/1/baseline.md, is missing. Only partial work exists: a measure script, characterization tests and a measure test. The measure scripts have a loop bug: `j` is never bounded or incremented, and `i < functions.length` is the condition used for `j`. The run ended as runner-terminal before a baseline was recorded.", "limitations": "Source-only review. There is no evidence of suite results, measurement output or advance_task_step. The cause of termination is unknown. Later-step files (plan, verification, review) were not expected at this step. This is not a final model failure or success.", "meanScore": null, "durationMs": 14803, "costUsd": 0.032992, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 7412, "cache_read_input_tokens": 700, "output_tokens": 327, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 7412, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 327, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 7412, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 7412 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 327, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 7412, "webSearchRequests": 0, "costUSD": 0.032992, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 7412, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-refactor-module / underdog-saluki-27b-iq2-mix\n\n**Composite: 4.0 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 49m 19s; budget used: 27.2%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at 3bfa0133424494ed)\"`; `facts.timing.timeToFirstArtifactMs: 529301`; `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":0,\"score\":6,\"scoreMax\":30,\"signals\":[],\"failReason\":\"task sourced from craftbook refactor-module has not finished: its gates have...` |\n| Output quality (25%) | 2 / 10 | The latest scenario gate reached 20% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 529301`; `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":0,\"score\":6,\"scoreMax\":30,\"signals\":[],\"failReason\":\"task sourced from craftbook refactor-module has not finished: its gates have...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.27165658626366723`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 529301`; `facts.outcome.durationMs: 2959043` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 2 + 0.2 * 10 + 0.15 * 10 = **4.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 11102 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 27990 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.76 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 13558 + 4746 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 1 / 0.33 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 14% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at 3bfa0133424494ed)\"`\n- `facts.timing.timeToFirstArtifactMs: 529301`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":0,\"score\":6,\"scoreMax\":30,\"signals\":[],\"failReason\":\"task sourced from craftbook refactor-module has not finished: its gates have...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 20% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 529301`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":0,\"score\":6,\"scoreMax\":30,\"signals\":[],\"failReason\":\"task sourced from craftbook refactor-module has not finished: its gates have...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.27165658626366723`\n- `facts.toolUse.totalToolCalls: 9`\n- `facts.timing.timeToFirstArtifactMs: 529301`\n- `facts.outcome.durationMs: 2959043`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 1094s — model busy but not delivering (hard digest stuck at 3bfa0133424494ed)\"`; `facts.timing.timeToFirstArtifactMs: 529301`; `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":0,\"score\":6,\"scoreMax\":30,\"signals\":[],\"failReason\":\"task sourced from craftbook refactor-module has not finished: its gates have...`\n- Quality: The latest scenario gate reached 20% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 529301`; `facts.sniff.latest: {\"filePath\":\"craftbook-refactor-module\",\"bytes\":0,\"score\":6,\"scoreMax\":30,\"signals\":[],\"failReason\":\"task sourced from craftbook refactor-module has not finished: its gates have...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.27165658626366723`; `facts.toolUse.totalToolCalls: 9`; `facts.timing.timeToFirstArtifactMs: 529301`; `facts.outcome.durationMs: 2959043`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **4/10**, eligible task failure; unchanged. Advisory **noArtifact**. No refactor or declared baseline/plan/change-notes/verification/review was delivered. The model runs2green tests, adds11useful characterization tests and gets13green, then spends the remaining work on unexecuted measurement helpers. Original duplicate implementations remain; normalize-name.js is absent.\n\n\n\n## Positives\n\nReal pre-change and characterization runs establish2/2 then13/13passes. New tests cover whitespace, all-caps, single letters, empty input, reportLine and reportSummary without changing production behavior.\n\n## Negatives and diagnosis\n\nNine calls never reach the actual extraction. A workspace edit mistakenly targets an artifact path, is rejected, and is corrected through write_artifact/copy. Both measurement helpers stop function bodies at the first closing brace, so nested loop bodies are truncated. The new measurement test asserts dupCount>=1, which would become wrong after successful deduplication; it has no recorded run. No source restructuring or claimed final artifact exists.\n\n## Performance and fit\n\nDecode 2.76tok/s, prefill 147.26tok/s; 4746generated tokens. Peak GPU 10628/12227MiB, process RSS 11102MiB, whole-system RAM 27990/31865MiB. Context 65536, 1slot, f16; max observed released slot 16557. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nUse the existing refactor role to record the already verified baseline and a simple explicit duplication count, then execute the smallest planned extraction. Keep diagnostic baseline assertions out of the permanent behavior suite.\n\n## Strategic experiment — not applied\n\nProvide reliable structural metrics without requiring a generated ad hoc parser. Evaluate whether artifact/workspace scope feedback helps models recover directly to the requested task.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/facts.json): pass, fixed 8.5, 61.95minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/facts.json): pass, fixed 8.5, 59.79minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/facts.json): fail, fixed 5, 26.51minutes; other historical baseline; contextual only.\n\nEarlier same-scenario trials reached a shared implementation; this attempt stops in baseline preparation. Codemod performs correct source edits but stalls on verification, while TDD completes the full evidence chain. No bridge/native failure appears here.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nTest bounded baseline preparation on refactor, codemod and small API extraction with generic metrics and unchanged behavior gates.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-developer-craftbook-bug-fix-tdd", "baselineCellId": "r1-saluki-cuda-developer-craftbook-bug-fix-tdd", "suite": "developer", "scenario": "craftbook-bug-fix-tdd", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T15:19:58.347Z", "finishedAt": "2026-11-10T16:20:12.197Z", "durationMs": 3613850, "success": true, "reason": "craftbook-bug-fix-tdd passed 28 deterministic craftbook checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 286288 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.26541720042415634 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 36 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 286288 }, { "source": "facts", "path": "outcome.durationMs", "value": 3613850 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "scenarioId": "craftbook-bug-fix-tdd", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 605, "peakRssMb": 15133, "peakCpuPercent": 1207.8, "firstAtMs": 1, "lastAtMs": 3599878 }, "gpu": { "available": true, "sampleCount": 605, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 605, "peakUsedMb": 30879, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 64955, "totalOutputTokens": 9013, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 64879, "todayTokensCached": 60990, "todayTokensOut": 3802, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 64879, "totalTokensCached": 60990, "totalTokensOut": 3802, "totalCost": 0, "medianOutputTokensPerSec": 2.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.9, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8511, "cachedInputTokens": 8004, "outputTokens": 69, "cost": 0, "durationMs": 300203, "outputTokensPerSec": 2.90899657670994, "at": "2026-11-10T16:20:02.061Z" }, "lastUpdated": "2026-11-10T16:20:02.061Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T16:20:06.741Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3602172, "meanTokensPerSec": 2.89, "genTokensPerSec": 2.89, "genTokens": 9013, "promptTokensPerSec": 171.96, "peakRssMb": 15133, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 64879, "output": 3802 } } } }, "craftbook": { "craftbookId": "bug-fix-tdd", "tasksObserved": 3, "craftbookTasks": 1, "reachedTerminal": true, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 6, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "orderly-pricing/1", "stepId": "reproduce", "activatedAt": "2026-11-10T15:20:05.970Z", "completedAt": "2026-11-10T15:28:38.584Z", "ms": 512614, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "b071a517-f99d-4d87-b841-3aa815566d4f" ] }, { "ref": "orderly-pricing/1", "stepId": "locate", "activatedAt": "2026-11-10T15:28:38.586Z", "completedAt": "2026-11-10T15:45:58.794Z", "ms": 1040208, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "26e3944e-d48f-4474-9125-1bf95d3c19e7" ] }, { "ref": "orderly-pricing/1", "stepId": "fix", "activatedAt": "2026-11-10T15:45:58.795Z", "completedAt": "2026-11-10T15:56:02.300Z", "ms": 603505, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "6fbf34e4-65dc-47f7-ba7d-63774da5deaa" ] }, { "ref": "orderly-pricing/1", "stepId": "validate", "activatedAt": "2026-11-10T15:56:02.302Z", "completedAt": "2026-11-10T16:02:19.720Z", "ms": 377418, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "6fbf34e4-65dc-47f7-ba7d-63774da5deaa" ] }, { "ref": "orderly-pricing/1", "stepId": "evaluate", "activatedAt": "2026-11-10T16:02:19.722Z", "completedAt": "2026-11-10T16:14:56.255Z", "ms": 756533, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "7b079970-b22d-4e12-b39b-13db4c7cd3b2" ] }, { "ref": "orderly-pricing/1", "stepId": "finish", "activatedAt": "2026-11-10T16:14:56.256Z", "completedAt": "2026-11-10T16:20:02.021Z", "ms": 305765, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "17cb9ea2-26fd-4578-b338-927d9723886f" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T15:20:04.320Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T15:20:04.174Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 558059.5 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "dawit": 2, "diem": 1, "ludovica": 1, "rekha": 1 }, "perTask": { "orderly-pricing/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.1430206298828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1430206298828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "craftbook-bug-fix-tdd passed 28 deterministic craftbook checks", "durationMs": 3613850, "timeoutMs": 13615734, "budgetUsedFraction": 0.26541720042415634 }, "timing": { "startedAt": "2026-11-10T15:19:58.347Z", "finishedAt": "2026-11-10T16:20:12.197Z", "timeToFirstArtifactMs": 286288, "timeToLastArtifactWriteMs": 3297933, "timeToFirstTokenMs": 78445, "firstTurnTtftMs": 63423, "timeToFirstToolCallMs": 136721 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Workflow Operator", "Developer", "Root-Cause Analyst", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 36, "byTool": { "read_files": 1, "write_file": 1, "run_package_script": 2, "write_artifact": 6, "read_artifact": 8, "read_file": 11, "list_dir": 1, "grep_files": 1, "replace_in_file": 1, "write_task_note": 2, "advance_task_step": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 13 }, "sniff": { "progression": [ { "atMs": 8315, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: package.json, src/pricing.js, tests/pricing.test.mjs, source/bug-report.md" }, { "atMs": 139066, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 6, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 287999, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 293777, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 299690, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 305839, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 311808, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 317768, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 323682, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 329582, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 335518, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 341892, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 347806, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 353767, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 359667, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 365563, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 371423, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 377326, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 383232, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 389116, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 395063, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 401457, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 407300, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 413212, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 419118, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 425032, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 430898, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 436829, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 442724, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 448613, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 454542, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 460476, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 466366, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 472333, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 478199, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 484138, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 490005, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 495960, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 501843, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 507863, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 513901, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 520016, "filePath": "craftbook-bug-fix-tdd", "bytes": 0, "score": 5, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 525905, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 532197, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 537903, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 544077, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 549976, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 555906, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 561822, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 567694, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 573628, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 579534, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 585455, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 591400, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 597285, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 603187, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 609207, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 615172, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 621078, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 627087, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 632997, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 638886, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 644830, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 650780, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 656530, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 662462, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 668367, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 674271, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 680200, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 686542, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 692470, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 698391, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 704294, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 710252, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 716644, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 722560, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 728506, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 734394, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 740317, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 746615, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 752543, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 758456, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 764336, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 770254, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 776125, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 782013, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 787921, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 793834, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 799767, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 805680, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 811606, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 817586, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 823497, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 829429, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 835319, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 841244, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 847141, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 853012, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 858927, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 864804, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 870727, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 876640, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 882474, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 888413, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 894324, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 900208, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 906107, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 912199, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 918107, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 924037, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 929932, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 935873, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 941756, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 947711, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 953638, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 959970, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 965889, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 971864, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 977819, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 983713, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 989620, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 995512, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1001743, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1007657, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1013570, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1019647, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1025531, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1031762, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1037683, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1043584, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1049463, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1055387, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1061810, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1067657, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1073592, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1079840, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1085870, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1091829, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1097706, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1103573, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1109497, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1115423, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1121826, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1127757, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1133655, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1139654, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1145536, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1151833, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1157765, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1163643, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1169564, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1175484, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1181838, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1187748, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1193698, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1199589, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1205536, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1211852, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1217768, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1223700, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1229573, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1235494, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1241903, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1247812, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1253742, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1259660, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1265604, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1271935, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1277811, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1283713, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1289714, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1295627, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1301524, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1307395, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1313304, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1319202, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1325099, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1331039, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1336957, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1342843, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1348726, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1354635, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1360621, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1366540, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1372470, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1378347, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1384242, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1390160, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1396009, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1401913, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1407818, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1413718, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1419602, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1425537, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1431480, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1437456, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1443382, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1449297, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1455196, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1461097, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1466997, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1472902, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1478792, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1484681, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1490610, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1496490, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1502329, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1508251, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1514138, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1520049, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1525997, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1531882, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1537785, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1543699, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1549607, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1555583, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 9, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1561829, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1567955, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1573836, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1579710, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1585570, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1591416, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1597287, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1603120, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1608998, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1614937, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1621036, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1626882, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1634288, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1640196, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1646102, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1652231, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1658209, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1664126, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1670019, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1675928, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1681835, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1687746, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 13, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1694345, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 16, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 1807650, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 17, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 2164637, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 21, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 2545628, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 24, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 3266564, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 26, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 3578108, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 27, "scoreMax": 28, "signals": [], "failReason": "task sourced from craftbook bug-fix-tdd has not finished: its gates have not accepted the result yet" }, { "atMs": 3607865, "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-bug-fix-tdd", "bytes": 1702, "score": 28, "scoreMax": 28, "signals": [], "failReason": null } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T15:19:58.352Z] [trial] id=craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty scenario=craftbook-bug-fix-tdd model=underdog-saluki-27b-iq2-mix", "[2026-11-10T15:19:58.352Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-bug-fix-tdd-WV6nTe", "[2026-11-10T15:19:58.352Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T15:19:58.352Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T15:19:58.352Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "[2026-11-10T15:19:58.355Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T15:19:58.360Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T15:19:58.367Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-bug-fix-tdd-WV6nTe", "[2026-11-10T15:19:58.367Z] [trial] throughput-scaled ceiling: 150m → 227m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T15:20:04.220Z] [trial] daemon spawned pid=32204 port=https://127.0.0.1:54166", "[2026-11-10T15:20:05.536Z] [trial] provider=llama-cpp generalistMode=auto packageScripts=on configured, firstRunCompleted=true", "[2026-11-10T15:20:05.554Z] [trial] meester=malika", "[2026-11-10T15:20:05.554Z] [trial] running scenario setup hook", "[2026-11-10T15:20:06.168Z] [trial] scenario setup complete", "[2026-11-10T15:20:06.170Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T15:20:06.232Z] [poll] initial digests hard=2f40e8afa8b2d79d soft=81692344bbc29eb2", "[2026-11-10T15:20:06.706Z] [poll] hard-progress: 2f40e8afa8b2d79d → 2298c211bd4c50b7 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-10T15:20:12.151Z] [poll] hard-progress: 2298c211bd4c50b7 → 9df738e144334b6a after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=5/0B)", "[2026-11-10T15:22:17.470Z] [poll] hard-progress: 9df738e144334b6a → 18d1884490a9ba9a after 125s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=151,stream=183) sniff=6/0B)", "[2026-11-10T15:24:46.417Z] [poll] hard-progress: 18d1884490a9ba9a → b17bf7cf25f59317 after 149s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=497,stream=745) sniff=5/0B)", "[2026-11-10T15:25:10.201Z] [poll] hard-progress: b17bf7cf25f59317 → 45a9e1a0581adfb2 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=514,stream=766) sniff=5/0B)", "[2026-11-10T15:28:38.517Z] [poll] hard-progress: 45a9e1a0581adfb2 → 48b21a487c08dc13 after 208s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1037,stream=1945) sniff=5/0B)", "[2026-11-10T15:28:44.324Z] [poll] hard-progress: 48b21a487c08dc13 → 3d6f8a2e77472e15 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=1037,stream=1945) sniff=9/1702B)", "[2026-11-10T15:29:26.082Z] [poll] hard-progress: 3d6f8a2e77472e15 → 1b35936592a76fd5 after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=1078,stream=1989) sniff=9/1702B)", "[2026-11-10T15:30:07.612Z] [poll] hard-progress: 1b35936592a76fd5 → 9302231636a8a9b4 after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=1126,stream=2063) sniff=9/1702B)" ] }, "metrics": { "process": { "sampleCount": 605, "peakRssMb": 15133, "peakCpuPercent": 1207.8, "firstAtMs": 1, "lastAtMs": 3599878 }, "gpu": { "available": true, "sampleCount": 605, "peakUtilPercent": 100, "peakMemUsedMb": 10640, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 605, "peakUsedMb": 30879, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 64955, "totalOutputTokens": 9013, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 64879, "todayTokensCached": 60990, "todayTokensOut": 3802, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 64879, "totalTokensCached": 60990, "totalTokensOut": 3802, "totalCost": 0, "medianOutputTokensPerSec": 2.9, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 2.9, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8511, "cachedInputTokens": 8004, "outputTokens": 69, "cost": 0, "durationMs": 300203, "outputTokensPerSec": 2.90899657670994, "at": "2026-11-10T16:20:02.061Z" }, "lastUpdated": "2026-11-10T16:20:02.061Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T16:20:06.741Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3602172, "meanTokensPerSec": 2.89, "genTokensPerSec": 2.89, "genTokens": 9013, "promptTokensPerSec": 171.96, "peakRssMb": 15133, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 64879, "output": 3802 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T15:19:58.347Z", "finishedAt": "2026-11-10T16:20:12.197Z", "timeToFirstArtifactMs": 286288, "timeToLastArtifactWriteMs": 3297933, "timeToFirstTokenMs": 78445, "firstTurnTtftMs": 63423, "timeToFirstToolCallMs": 136721 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 8, "activationEvents": 5, "completed": 6, "gateApprovals": 6, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "orderly-pricing/1", "stepId": "reproduce", "activatedAt": "2026-11-10T15:20:05.970Z", "completedAt": "2026-11-10T15:28:38.584Z", "ms": 512614, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "b071a517-f99d-4d87-b841-3aa815566d4f" ] }, { "ref": "orderly-pricing/1", "stepId": "locate", "activatedAt": "2026-11-10T15:28:38.586Z", "completedAt": "2026-11-10T15:45:58.794Z", "ms": 1040208, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "26e3944e-d48f-4474-9125-1bf95d3c19e7" ] }, { "ref": "orderly-pricing/1", "stepId": "fix", "activatedAt": "2026-11-10T15:45:58.795Z", "completedAt": "2026-11-10T15:56:02.300Z", "ms": 603505, "gateRejections": 0, "toolCalls": 8, "sessionIds": [ "6fbf34e4-65dc-47f7-ba7d-63774da5deaa" ] }, { "ref": "orderly-pricing/1", "stepId": "validate", "activatedAt": "2026-11-10T15:56:02.302Z", "completedAt": "2026-11-10T16:02:19.720Z", "ms": 377418, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "6fbf34e4-65dc-47f7-ba7d-63774da5deaa" ] }, { "ref": "orderly-pricing/1", "stepId": "evaluate", "activatedAt": "2026-11-10T16:02:19.722Z", "completedAt": "2026-11-10T16:14:56.255Z", "ms": 756533, "gateRejections": 0, "toolCalls": 10, "sessionIds": [ "7b079970-b22d-4e12-b39b-13db4c7cd3b2" ] }, { "ref": "orderly-pricing/1", "stepId": "finish", "activatedAt": "2026-11-10T16:14:56.256Z", "completedAt": "2026-11-10T16:20:02.021Z", "ms": 305765, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "17cb9ea2-26fd-4578-b338-927d9723886f" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T15:20:04.320Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T15:20:04.174Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 558059.5 }, "sessions": { "total": 5, "taskScoped": 5, "byGezel": { "dawit": 2, "diem": 1, "ludovica": 1, "rekha": 1 }, "perTask": { "orderly-pricing/1": 5 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.625, "maxMessages": 4, "maxContextFill": 0.1430206298828125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1430206298828125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-developer-craftbook-bug-fix-tdd", "baselineCellId": "r1-saluki-cuda-developer-craftbook-bug-fix-tdd", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "scenarioId": "craftbook-bug-fix-tdd", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T17:06:02.428Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/fix-notes.md", "artifactBytes": 2828, "artifactSha256": "e020342999a81f677a63fa533e74a4ca73e71ec0ef595345ba2d4125568a8404", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/bu/bug-fix-tdd/versions/2.0.7/test.json", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ], "scoreAxes": { "Repro-first discipline": 8, "Fix minimality": 9, "Honest verification": 7 }, "meanScore": 8, "justification": "A real boundary test fails80≠68 before the one-character >→>= fix, then all4tests pass. Notes and review preserve the causal chain and actual receipts. Some verification prose overstates universal coverage; the diagnosis write first hits grounding rejections. The full workflow finishes despite slow decode.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/fix-notes.md", "status": "scored", "bytes": 2828, "sha256": "e020342999a81f677a63fa533e74a4ca73e71ec0ef595345ba2d4125568a8404", "reviewedBytes": 2828, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/repro.md", "status": "notApplicable", "bytes": 1726, "sha256": "0d89f4890d627f4ab2ff9d270ab7a94b44a2d54315d11d80abe09b2ca27aa83f", "reviewedBytes": 1726, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/diagnosis.md", "status": "notApplicable", "bytes": 3464, "sha256": "19a1db56dd00827aa5415cbf9c4aad404af175e03881858ce25dd553bc490ea9", "reviewedBytes": 3464, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/validation.md", "status": "notApplicable", "bytes": 1555, "sha256": "8241254f3335b568bd5ed13719e1f3dd0750d70268e33b82a9fa7ce4ce4e8ae6", "reviewedBytes": 1555, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/review.md", "status": "notApplicable", "bytes": 3333, "sha256": "9b6e6d60a4bc6813acd16369688c17a8fc022f80f837e3842b9aa30a099bf3b8", "reviewedBytes": 3333, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/workspace/orderly-pricing/src/pricing.js", "status": "notApplicable", "bytes": 474, "sha256": "fad23b0c7929def833f4dcb9c3557a77ccb3f3ad92b3be97e462fea4e0a56921", "reviewedBytes": 474, "truncated": false, "role": "modifiedSource" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/workspace/orderly-pricing/tests/pricing.test.mjs", "status": "notApplicable", "bytes": 597, "sha256": "59d0ebcb35b11217b4b68e85aeedec267e2471718314f2eb8df2325724ae0d01", "reviewedBytes": 597, "truncated": false, "role": "regressionTest" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T15:20:11.433Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T15:20:15.255Z INFO [chat] [llama-server] launch {\"pid\":40520,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T16:20:11.386Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-40520-1794327611386\",\"pid\":40520,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":3596131,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 12700, "peakGpuMiB": 10640, "peakProcessRssMiB": 15133, "peakSystemRamMiB": 30879, "decodeTokensPerSec": 2.89, "prefillTokensPerSec": 171.96, "generatedTokens": 9013, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1430206298828125 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:06:59.059Z", "reviewId": "120", "cellId": "r3-saluki-cuda-developer-craftbook-bug-fix-tdd", "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "scenarioId": "craftbook-bug-fix-tdd", "status": "scored", "axes": [ { "name": "Repro-first discipline", "description": "A failing test was genuinely written and run before the fix, in the project's real test layout, and the red evidence quotes a real run rather than a claim." }, { "name": "Fix minimality", "description": "The change is the smallest correct edit at the real defect site; no special-casing of the reported input, no unrelated churn." }, { "name": "Honest verification", "description": "Validation re-ran the original reproduction, distinguishes verified results from anything unverified, and the review graded evidence rather than summaries." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "baselineCellId": "r1-saluki-cuda-developer-craftbook-bug-fix-tdd", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "521b5b071e724478ec1c72e619b74d76779ac1c36b87721dfbd7c5b7b6afca37", "evidenceSha256": "a400ef48153bd0366bca6176f57622cb537196e85beb5c7ad39625e16f51d5c9", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/fix-notes.md", "bytes": 2828, "sha256": "e020342999a81f677a63fa533e74a4ca73e71ec0ef595345ba2d4125568a8404", "shownBytes": 2828, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/repro.md", "bytes": 1726, "sha256": "0d89f4890d627f4ab2ff9d270ab7a94b44a2d54315d11d80abe09b2ca27aa83f", "shownBytes": 1726, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/diagnosis.md", "bytes": 3464, "sha256": "19a1db56dd00827aa5415cbf9c4aad404af175e03881858ce25dd553bc490ea9", "shownBytes": 3464, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/validation.md", "bytes": 1555, "sha256": "8241254f3335b568bd5ed13719e1f3dd0750d70268e33b82a9fa7ce4ce4e8ae6", "shownBytes": 1555, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/review.md", "bytes": 3333, "sha256": "9b6e6d60a4bc6813acd16369688c17a8fc022f80f837e3842b9aa30a099bf3b8", "shownBytes": 3333, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/workspace/orderly-pricing/src/pricing.js", "bytes": 474, "sha256": "fad23b0c7929def833f4dcb9c3557a77ccb3f3ad92b3be97e462fea4e0a56921", "shownBytes": 474, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/workspace/orderly-pricing/tests/pricing.test.mjs", "bytes": 597, "sha256": "59d0ebcb35b11217b4b68e85aeedec267e2471718314f2eb8df2325724ae0d01", "shownBytes": 597, "truncated": false } ], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty\\workspace\\orderly-pricing\\source\\bug-report.md", "bytes": 313, "sha256": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "shownBytes": 313, "truncated": false } ], "scoreAxes": { "Repro-first discipline": 8, "Fix minimality": 9, "Honest verification": 7 }, "justification": "The regression test sits in tests/pricing.test.mjs, and repro.md quotes a failing run (exit 1, 80 !== 68). The fix is one character (> to >=) at src/pricing.js:8, with no special-casing. validation.md quotes a green run (4/4). Fix-notes is candid about unverified siblings. The review leans on recorded docs, and \"nothing remains unverified\" overclaims.", "limitations": "Run receipts are quoted in the artifacts, not independently visible. I could not confirm the exit codes or run timing. The fix-notes admits no shell was available in that step. I did not re-run anything, so the reviewer's independence is unverified.", "meanScore": 8, "durationMs": 16696, "costUsd": 0.0437204, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 9346, "cache_read_input_tokens": 824, "output_tokens": 625, "output_tokens_details": { "thinking_tokens": 259 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 9346, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 625, "cache_read_input_tokens": 824, "cache_creation_input_tokens": 9346, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9346 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 625, "cacheReadInputTokens": 824, "cacheCreationInputTokens": 9346, "webSearchRequests": 0, "costUSD": 0.0437204, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 259, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 9346, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-bug-fix-tdd / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 60m 14s; budget used: 26.5%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 286288`; `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1702,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.26541720042415634`; `facts.toolUse.totalToolCalls: 36`; `facts.timing.timeToFirstArtifactMs: 286288`; `facts.outcome.durationMs: 3613850` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15133 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10640 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30879 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.89 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 64955 + 9013 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 8 → 6 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 558s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 5 / 0.63 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 14% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 286288`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1702,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.26541720042415634`\n- `facts.toolUse.totalToolCalls: 36`\n- `facts.timing.timeToFirstArtifactMs: 286288`\n- `facts.outcome.durationMs: 3613850`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 286288`; `facts.sniff.latest: {\"filePath\":\"craftbook-bug-fix-tdd\",\"bytes\":1702,\"score\":28,\"scoreMax\":28,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.26541720042415634`; `facts.toolUse.totalToolCalls: 36`; `facts.timing.timeToFirstArtifactMs: 286288`; `facts.outcome.durationMs: 3613850`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.5/10**, pass; unchanged. Advisory **8/10**. A real boundary test fails80≠68 before the one-character >→>= fix, then all4tests pass. Notes and review preserve the causal chain and actual receipts. Some verification prose overstates universal coverage; the diagnosis write first hits grounding rejections. The full workflow finishes despite slow decode.\n\nExact declared axes: Repro-first discipline=8; Fix minimality=9; Honest verification=7.\n\n## Positives\n\nRecorded chronology is test write15:24:44, red run15:25:03, production fix15:48:08, green run15:58:34. The minimal generic threshold fix does not special-case price or quantity; the original three tests remain. All five required workflow artifacts are present, and review opens actual sources.\n\n## Negatives and diagnosis\n\nNothing remains unverified overstates the small suite, and the reviewer calls existing coverage proof that all sibling behavior is unchanged. Fix notes list only the production file in Files touched although the workflow also adds a regression test (described elsewhere). Diagnosis initially encounters grounding rejections for line references and semantic descriptions; revised text succeeds. Thirty-six calls and9013tokens for this small repair impose substantial latency.\n\n## Performance and fit\n\nDecode 2.89tok/s, prefill 171.96tok/s; 9013generated tokens. Peak GPU 10640/12227MiB, process RSS 15133MiB, whole-system RAM 30879/31865MiB. Context 65536, 1slot, f16; max observed released slot 12700. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing reviewer role, distinguish tested cases from source-level reasoning and list workflow-wide test/source changes. Preserve the exact red-before-fix and green-after-fix receipts.\n\n## Strategic experiment — not applied\n\nCarry durable test receipts and file-change inventories across steps to avoid repeated narration/read overhead while retaining independent review. Validate on non-boundary and multi-file bugs.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/facts.json): pass, fixed 8.5, 25.14minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/facts.json): pass, fixed 8.5, 23.04minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/facts.json): fail, fixed 5.5, 44.23minutes; other historical baseline; contextual only.\n\nEarlier same-scenario trials also complete red-to-green; this confirms one genuine developer success in the slower phase. Route-only bug-fix selection cannot substitute for this evidence. Squisq no-edit and refactor baseline stalls show limits on broader competence.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nMeasure a compact evidence-carrying TDD handoff across boundary, state-machine and cross-file bugs in a separate baseline; preserve current scores and artifact contents.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-complex-work-craftbook-find-vs-create", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-find-vs-create", "suite": "complex-work", "scenario": "craftbook-find-vs-create", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T16:22:59.367Z", "finishedAt": "2026-11-10T16:26:26.778Z", "durationMs": 207411, "success": true, "reason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 5, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 50% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.11424890287760868 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 1 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 207411 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 8.8, "band": "ship-ready" }, "facts": { "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "scenarioId": "craftbook-find-vs-create", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 33, "peakRssMb": 9702, "peakCpuPercent": 1195.9, "firstAtMs": 1, "lastAtMs": 193150 }, "gpu": { "available": true, "sampleCount": 33, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 26641, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11639, "totalOutputTokens": 264, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T16:26:20.201Z" } }, "source": "engine-log" }, "derived": { "durationMs": 194624, "meanTokensPerSec": 2.79, "genTokensPerSec": 2.79, "genTokens": 264, "promptTokensPerSec": 168.47, "peakRssMb": 9702, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shop-site-pages/1", "stepId": "scope", "activatedAt": "2026-11-10T16:26:15.307Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T16:23:05.270Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T16:23:05.117Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "bastien": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.15106201171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.15106201171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "knowledge://bendyline/handboek/whats-new/1.26234", "knowledge://bendyline/handboek/craftbook/accessibility-retrofit", "knowledge://bendyline/handboek/craftbook/podcast-chaptering" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "durationMs": 207411, "timeoutMs": 1815431, "budgetUsedFraction": 0.11424890287760868 }, "timing": { "startedAt": "2026-11-10T16:22:59.367Z", "finishedAt": "2026-11-10T16:26:26.778Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 100213, "firstTurnTtftMs": 80344, "timeToFirstToolCallMs": 196089 }, "team": { "totalGezelsCreated": 5, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 1, "byTool": { "start_work": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/shop-site-pages/pages/index.html", "finalBytes": 168, "growth": [] }, { "path": "workspace/shop-site-pages/pages/shipping.html", "finalBytes": 165, "growth": [] } ], "imageFiles": [], "otherFileCount": 1 }, "sniff": { "progression": [ { "atMs": 7994, "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } ], "latest": { "filePath": "craftbook-find-vs-create", "bytes": 3, "score": 1, "scoreMax": 2, "signals": [], "failReason": "no task sourced from the bundled \"seo-meta-pack\" craftbook exists yet — search the recipe library for the page-metadata recipe and invoke it on this project" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T16:22:59.371Z] [trial] id=craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq scenario=craftbook-find-vs-create model=underdog-saluki-27b-iq2-mix", "[2026-11-10T16:22:59.371Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "[2026-11-10T16:22:59.371Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T16:22:59.371Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-find-vs-create-MEZe6U", "[2026-11-10T16:22:59.375Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T16:22:59.371Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T16:22:59.379Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T16:22:59.386Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-find-vs-create-MEZe6U", "[2026-11-10T16:22:59.387Z] [trial] throughput-scaled ceiling: 20m → 30m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T16:23:05.164Z] [trial] daemon spawned pid=46864 port=https://127.0.0.1:58716", "[2026-11-10T16:23:06.685Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T16:23:06.700Z] [trial] meester=bastien", "[2026-11-10T16:23:06.700Z] [trial] running scenario setup hook", "[2026-11-10T16:23:07.038Z] [trial] scenario setup complete", "[2026-11-10T16:23:07.039Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T16:23:07.116Z] [poll] initial digests hard=a87aa621df73fb55 soft=3ba669f9a2674157", "[2026-11-10T16:23:07.498Z] [poll] hard-progress: a87aa621df73fb55 → f929ef85071d0541 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/3B)", "[2026-11-10T16:26:19.727Z] [poll] terminal: success=true reason=task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored", "[2026-11-10T16:26:20.225Z] [capture] dumped 1 session(s)", "[2026-11-10T16:26:20.228Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-10T16:26:20.282Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)", "[2026-11-10T16:26:26.777Z] [trial] daemon shut down", "[2026-11-10T16:26:26.787Z] [trial] PASS duration=207411ms reason=task shop-site-pages/1 was sourced from the bundled \"seo-meta-pack\" craftbook with no new local craftbook authored" ] }, "metrics": { "process": { "sampleCount": 33, "peakRssMb": 9702, "peakCpuPercent": 1195.9, "firstAtMs": 1, "lastAtMs": 193150 }, "gpu": { "available": true, "sampleCount": 33, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 33, "peakUsedMb": 26641, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 11639, "totalOutputTokens": 264, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T16:26:20.201Z" } }, "source": "engine-log" }, "derived": { "durationMs": 194624, "meanTokensPerSec": 2.79, "genTokensPerSec": 2.79, "genTokens": 264, "promptTokensPerSec": 168.47, "peakRssMb": 9702, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T16:22:59.367Z", "finishedAt": "2026-11-10T16:26:26.778Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 100213, "firstTurnTtftMs": 80344, "timeToFirstToolCallMs": 196089 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "shop-site-pages/1", "stepId": "scope", "activatedAt": "2026-11-10T16:26:15.307Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T16:23:05.270Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T16:23:05.117Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "bastien": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.15106201171875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.15106201171875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-complex-work-craftbook-find-vs-create", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-find-vs-create", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "scenarioId": "craftbook-find-vs-create", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-10T17:06:04.288Z", "artifactPath": null, "artifactBytes": 0, "artifactSha256": null, "axisSource": "evals/src/craftbooks/authoring/find-vs-create.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "A single start_work call selects the existing seo-meta-pack recipe and dispatches its scope step without creating a new recipe. This is successful find-versus-create routing only; no authored page metadata or finished downstream workflow is claimed. No declared artifact axes apply.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T16:23:08.896Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T16:23:21.295Z INFO [chat] [llama-server] launch {\"pid\":17756,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T16:26:26.006Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-17756-1794327986006\",\"pid\":17756,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":184711,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 11902, "peakGpuMiB": 10628, "peakProcessRssMiB": 9702, "peakSystemRamMiB": 26641, "decodeTokensPerSec": 2.79, "prefillTokensPerSec": 168.47, "generatedTokens": 264, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.15106201171875 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:07:01.135Z", "reviewId": "121", "cellId": "r3-saluki-cuda-complex-work-craftbook-find-vs-create", "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "scenarioId": "craftbook-find-vs-create", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-find-vs-create", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "8fe72ee5307d558b60510f02edefdd222505cf967ee1742f53588100f34c9e77", "evidenceSha256": "1de0c7f11472a2487573303a94f45c446b726f767607c1bb3ac2301b7525a862", "artifactFiles": [], "contextFiles": [], "scoreAxes": {}, "justification": "No declared axes and no artifacts were supplied. The executionDisposition is runner-terminal, and the evidence has no transcript, tool calls or workspace outputs. I can't tell whether suggest_craftbook or invoke_craftbook were called, or whether pages were edited. No qualitative judgment on correctness is possible.", "limitations": "Source-only review with empty artifacts and context files. I saw no tool-call trace, no page metadata outputs and no project workspace state. Whether the craftbook workflow was followed, or whether start_project was avoided, is unobserved and unverified.", "meanScore": null, "durationMs": 14170, "costUsd": 0.009114, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1485, "cache_read_input_tokens": 700, "output_tokens": 310, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1485, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 310, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1485, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1485 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 310, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1485, "webSearchRequests": 0, "costUSD": 0.009114, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1485, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-find-vs-create / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.8 / 10** (band: ship-ready)\n**Outcome: success** (duration: 3m 27s; budget used: 11.4%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 5 / 10 | The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.11424890287760868`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 207411` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 10 + 0.25 * 5 + 0.2 * 10 + 0.15 * 10 = **8.8**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9702 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 26641 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.79 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 11639 + 264 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 15% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 50% of its measured signals.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.11424890287760868`\n- `facts.toolUse.totalToolCalls: 1`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 207411`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The latest scenario gate reached 50% of its measured signals. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-find-vs-create\",\"bytes\":3,\"score\":1,\"scoreMax\":2,\"signals\":[],\"failReason\":\"no task sourced from the bundled \\\"seo-meta-pack\\\" craftbook exists yet — sear...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.11424890287760868`; `facts.toolUse.totalToolCalls: 1`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 207411`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.8/10**, pass; unchanged. Advisory **notApplicable**. A single start_work call selects the existing seo-meta-pack recipe and dispatches its scope step without creating a new recipe. This is successful find-versus-create routing only; no authored page metadata or finished downstream workflow is claimed. No declared artifact axes apply.\n\n\n\n## Positives\n\nThe model reuses the relevant bundled recipe directly, meeting the route-selection contract without duplicating catalog content.\n\n## Negatives and diagnosis\n\nThe scenario terminates at task creation, so it does not establish that page metadata was written, reviewed or published. The fixed8.8 includes a stale pre-dispatch sniff component; the terminal result establishes successful selection.\n\n## Performance and fit\n\nDecode 2.79tok/s, prefill 168.47tok/s; 264generated tokens. Peak GPU 10628/12227MiB, process RSS 9702MiB, whole-system RAM 26641/31865MiB. Context 65536, 1slot, f16; max observed released slot 11902. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nKeep the existing authoring/routing role explicit that work has started and avoid announcing completion of the downstream deliverables.\n\n## Strategic experiment — not applied\n\nSeparate reuse/selection accuracy from recipe execution success in cross-suite reporting. Validate genuinely missing recipes and ambiguous matches as distinct cases.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/facts.json): pass, fixed 8.8, 1.93minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/facts.json): pass, fixed 8.8, 1.92minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/facts.json): pass, fixed 8.8, 0.82minutes; other historical baseline; contextual only.\n\nThe same-scenario routing results and dev-craftbook-routing are narrow selection measurements. The following edit-midtask trial tests actual recipe modification and stalls; this contrast limits claims about general authoring competence.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nEvaluate selection and subsequent execution as distinct outcomes on a new mixed existing/missing-recipe set without altering this frozen campaign.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-complex-work-craftbook-edit-midtask", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-edit-midtask", "suite": "complex-work", "scenario": "craftbook-edit-midtask", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T16:28:41.235Z", "finishedAt": "2026-11-10T16:52:32.888Z", "durationMs": 1431653, "success": false, "reason": "no real progress for 911s — model busy but not delivering (hard digest stuck at d8ddef3189ab423b)", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-edit-midtask", "score": 1, "bytes": 0, "deliverableMissing": true, "milestones": 0, "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 0, "ruleId": "C_WATCHDOG_LOW_OR_NONE", "summary": "A watchdog ended the trial without evidence that at least half of the gate was reached.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "no real progress for 911s — model busy but not delivering (hard digest stuck at d8ddef3189ab423b)" }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-edit-midtask", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" } } ] }, "quality": { "score": 0, "ruleId": "Q_NO_PRODUCED_OUTPUT", "summary": "No authored artifact event or scored sniff output was observed.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-edit-midtask", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" } } ] }, "efficiency": { "score": 10, "ruleId": "E_LOW_BUDGET_LOW_CALLS", "summary": "The trial used at most half its budget and no more than ten tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.35048987445895924 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 5 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": null }, { "source": "facts", "path": "outcome.durationMs", "value": 1431653 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 3.5, "band": "framework-gap" }, "facts": { "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "scenarioId": "craftbook-edit-midtask", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 238, "peakRssMb": 9734, "peakCpuPercent": 1212.2, "firstAtMs": 1, "lastAtMs": 1417759 }, "gpu": { "available": true, "sampleCount": 238, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 238, "peakUsedMb": 28218, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 18778, "totalOutputTokens": 946, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T16:52:26.402Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1419408, "meanTokensPerSec": 2.64, "genTokensPerSec": 2.64, "genTokens": 946, "promptTokensPerSec": 146.69, "peakRssMb": 9734, "peakGpuUtilPercent": 100, "billing": null } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "quarterly-brief/1", "stepId": "draft", "activatedAt": "2026-11-10T16:28:48.222Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T16:28:47.020Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T16:28:46.888Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "reza-2": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.21649169921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.21649169921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 0, "tasks": 3, "tasksWithReferences": 0, "items": 0, "citations": [], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "no real progress for 911s — model busy but not delivering (hard digest stuck at d8ddef3189ab423b)", "durationMs": 1431653, "timeoutMs": 4084720, "budgetUsedFraction": 0.35048987445895924 }, "timing": { "startedAt": "2026-11-10T16:28:41.235Z", "finishedAt": "2026-11-10T16:52:32.888Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 134477, "firstTurnTtftMs": 123347, "timeToFirstToolCallMs": 160929 }, "team": { "totalGezelsCreated": 6, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 5, "byTool": { "get_task": 1, "list_dir": 1, "read_file": 1, "read_task_notes": 1, "craftbook_read": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 2 }, "sniff": { "progression": [ { "atMs": 7370, "filePath": "craftbook-edit-midtask", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" } ], "latest": { "filePath": "craftbook-edit-midtask", "bytes": 0, "score": 1, "scoreMax": 4, "signals": [], "failReason": "task quarterly-brief/1 still carries the original failing step prompt verbatim — read the task's craftbook, find what the final gate demands, and rewrite that step's prompt to instruct it" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T16:28:41.240Z] [trial] id=craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67 scenario=craftbook-edit-midtask model=underdog-saluki-27b-iq2-mix", "[2026-11-10T16:28:41.241Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-edit-midtask-V1AFAQ", "[2026-11-10T16:28:41.241Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "[2026-11-10T16:28:41.241Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T16:28:41.244Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T16:28:41.241Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T16:28:41.248Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T16:28:41.256Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-edit-midtask-V1AFAQ", "[2026-11-10T16:28:41.256Z] [trial] throughput-scaled ceiling: 45m → 68m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T16:28:46.913Z] [trial] daemon spawned pid=1424 port=https://127.0.0.1:61645", "[2026-11-10T16:28:47.998Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T16:28:48.014Z] [trial] meester=cosima", "[2026-11-10T16:28:48.015Z] [trial] running scenario setup hook", "[2026-11-10T16:28:48.482Z] [trial] scenario setup complete", "[2026-11-10T16:28:48.484Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T16:28:48.570Z] [poll] initial digests hard=76937768184f94d6 soft=d81c4ebcd8afaa39", "[2026-11-10T16:28:48.766Z] [poll] hard-progress: 76937768184f94d6 → f45ac24fb57818d6 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T16:31:24.960Z] [poll] hard-progress: f45ac24fb57818d6 → 56bc6cf6ffc5e439 after 156s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=44,stream=47) sniff=1/0B)", "[2026-11-10T16:32:54.007Z] [poll] hard-progress: 56bc6cf6ffc5e439 → c11d4918b12ea970 after 89s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=208,stream=227) sniff=1/0B)", "[2026-11-10T16:37:15.383Z] [poll] hard-progress: c11d4918b12ea970 → d8ddef3189ab423b after 261s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=831,stream=871) sniff=1/0B)", "[2026-11-10T16:52:25.937Z] [poll] no-progress (hard): no real progress for 911s (threshold=908s); model busy but not delivering — failing trial", "[2026-11-10T16:52:26.471Z] [capture] dumped 1 session(s)", "[2026-11-10T16:52:26.491Z] [capture] snapshotted telemetry for 1 session(s)", "[2026-11-10T16:52:26.492Z] [capture] INCOMPLETE TRANSCRIPT for 1 session(s) — reza-2: 0 persisted vs 5 recorded. Assistant turns commit at turn END, so a mid-turn termination loses them. Triage from session-telemetry.json and daemon.log, NOT from the empty transcript.", "[2026-11-10T16:52:26.576Z] [capture] snapshotted artifacts/workspace/history for 3 project(s)" ] }, "metrics": { "process": { "sampleCount": 238, "peakRssMb": 9734, "peakCpuPercent": 1212.2, "firstAtMs": 1, "lastAtMs": 1417759 }, "gpu": { "available": true, "sampleCount": 238, "peakUtilPercent": 100, "peakMemUsedMb": 10628, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 238, "peakUsedMb": 28218, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 18778, "totalOutputTokens": 946, "totalCostUsd": 0, "rawProviders": { "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T16:52:26.402Z" } }, "source": "engine-log" }, "derived": { "durationMs": 1419408, "meanTokensPerSec": 2.64, "genTokensPerSec": 2.64, "genTokens": 946, "promptTokensPerSec": 146.69, "peakRssMb": 9734, "peakGpuUtilPercent": 100, "billing": null } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T16:28:41.235Z", "finishedAt": "2026-11-10T16:52:32.888Z", "timeToFirstArtifactMs": null, "timeToLastArtifactWriteMs": null, "timeToFirstTokenMs": 134477, "firstTurnTtftMs": 123347, "timeToFirstToolCallMs": 160929 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 3, "activationEvents": 0, "completed": 0, "gateApprovals": 0, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "quarterly-brief/1", "stepId": "draft", "activatedAt": "2026-11-10T16:28:48.222Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T16:28:47.020Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T16:28:46.888Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": null }, "sessions": { "total": 1, "taskScoped": 0, "byGezel": { "reza-2": 1 }, "perTask": {}, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0, "maxMessages": 1, "maxContextFill": 0.21649169921875, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.21649169921875 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-complex-work-craftbook-edit-midtask", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-edit-midtask", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "scenarioId": "craftbook-edit-midtask", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "noArtifact", "provisional": false, "reviewedAt": "2026-11-10T17:06:06.297Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/workspace/quarterly-brief/out/summary.md", "artifactBytes": 0, "artifactSha256": null, "axisSource": "evals/src/craftbooks/authoring/edit-midtask.ts; no declared judge axes", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "Five read-only calls inspect the task, notes and craftbook, then the model stalls without a recipe edit, verification step or summary. The required craftbook_write/craftbook_add_step tools were present. The original failing prompt and two-step recipe remain; no native or mock-tool fault is evident.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/workspace/quarterly-brief/out/summary.md", "status": "noArtifact", "bytes": 0, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T16:28:50.388Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T16:28:55.827Z INFO [chat] [llama-server] launch {\"pid\":38056,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T16:52:32.078Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-38056-1794329552078\",\"pid\":38056,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":1416251,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 22583, "peakGpuMiB": 10628, "peakProcessRssMiB": 9734, "peakSystemRamMiB": 28218, "decodeTokensPerSec": 2.64, "prefillTokensPerSec": 146.69, "generatedTokens": 946, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.21649169921875 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:07:09.635Z", "reviewId": "122", "cellId": "r3-saluki-cuda-complex-work-craftbook-edit-midtask", "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "scenarioId": "craftbook-edit-midtask", "status": "noArtifact", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-edit-midtask", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "78f378dc6fdc73a06ddeae011947e8eac3b3e7c7f6d6eef872e6fd0b018cdc33", "evidenceSha256": "13d8cbd3ac4c06f6567d5d4f3cd6205da1b949c1b84b98297a55566c2f331149", "artifactFiles": [], "contextFiles": [ { "path": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67\\workspace\\quarterly-brief\\data\\notes.txt", "bytes": 544, "sha256": "949c911a4bbdf1083c68bbe94d539edda30128a353f4028a07a65bb717752a47", "shownBytes": 544, "truncated": false } ], "scoreAxes": {}, "justification": "The declared deliverable workspace/quarterly-brief/out/summary.md was absent at closeout, and no artifacts were captured. The only evidence is the input notes.txt, which shows source data but no craftbook edits, gate results, or task completion. The cause of the missing output can't be determined.", "limitations": "No transcript, craftbook contents, gate definition, tool-call logs, or run outcome were provided. I can't tell whether the craftbook was fixed, whether the task was driven to completion, or why the run ended. Nothing here supports a success or failure judgment on the model.", "meanScore": null, "durationMs": 14116, "costUsd": 0.010598, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 1866, "cache_read_input_tokens": 700, "output_tokens": 306, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 1866, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 306, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 1866, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 1866 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 306, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 1866, "webSearchRequests": 0, "costUSD": 0.010598, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 1866, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-edit-midtask / underdog-saluki-27b-iq2-mix\n\n**Composite: 3.5 / 10** (band: framework-gap)\n**Outcome: model-stuck** (duration: 23m 52s; budget used: 35.0%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 0 / 10 | A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 911s — model busy but not delivering (hard digest stuck at d8ddef3189ab423b)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"task quarterly-brief/1 still carries the original failing step prompt verbatim —...` |\n| Output quality (25%) | 0 / 10 | No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"task quarterly-brief/1 still carries the original failing step prompt verbatim —...` |\n| Process efficiency (20%) | 10 / 10 | The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.35048987445895924`; `facts.toolUse.totalToolCalls: 5`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 1431653` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 0 + 0.25 * 0 + 0.2 * 10 + 0.15 * 10 = **3.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 9734 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10628 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 28218 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 2.64 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 18778 + 946 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 3 → 0 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | — | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 0 / 0.00 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 22% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 0 / 0 of 3 / 0 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_LOW_OR_NONE\n\nA watchdog ended the trial without evidence that at least half of the gate was reached.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"no real progress for 911s — model busy but not delivering (hard digest stuck at d8ddef3189ab423b)\"`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"task quarterly-brief/1 still carries the original failing step prompt verbatim —...`\n\n### Output quality - Q_NO_PRODUCED_OUTPUT\n\nNo authored artifact event or scored sniff output was observed.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"task quarterly-brief/1 still carries the original failing step prompt verbatim —...`\n\n### Process efficiency - E_LOW_BUDGET_LOW_CALLS\n\nThe trial used at most half its budget and no more than ten tool calls.\n\n- `facts.outcome.budgetUsedFraction: 0.35048987445895924`\n- `facts.toolUse.totalToolCalls: 5`\n- `facts.timing.timeToFirstArtifactMs: null`\n- `facts.outcome.durationMs: 1431653`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial without evidence that at least half of the gate was reached. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"no real progress for 911s — model busy but not delivering (hard digest stuck at d8ddef3189ab423b)\"`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"task quarterly-brief/1 still carries the original failing step prompt verbatim —...`\n- Quality: No authored artifact event or scored sniff output was observed. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: null`; `facts.sniff.latest: {\"filePath\":\"craftbook-edit-midtask\",\"bytes\":0,\"score\":1,\"scoreMax\":4,\"signals\":[],\"failReason\":\"task quarterly-brief/1 still carries the original failing step prompt verbatim —...`\n- Efficiency: The trial used at most half its budget and no more than ten tool calls. `facts.outcome.budgetUsedFraction: 0.35048987445895924`; `facts.toolUse.totalToolCalls: 5`; `facts.timing.timeToFirstArtifactMs: null`; `facts.outcome.durationMs: 1431653`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **3.5/10**, eligible task failure; unchanged. Advisory **noArtifact**. Five read-only calls inspect the task, notes and craftbook, then the model stalls without a recipe edit, verification step or summary. The required craftbook_write/craftbook_add_step tools were present. The original failing prompt and two-step recipe remain; no native or mock-tool fault is evident.\n\n\n\n## Positives\n\nThe task’s actual live craftbook is read in markdown form, exposing the final gate’s Source Register requirement and its omission from the prompt. The model does not weaken the gate or create a replacement task.\n\n## Negatives and diagnosis\n\nNo change follows inspection. The exact task requires editing the existing recipe, adding a verification step and completing the summary; none occurs. This cannot be counted as successful authoring merely because a large86-tool roster loaded.\n\n## Performance and fit\n\nDecode 2.64tok/s, prefill 146.69tok/s; 946generated tokens. Peak GPU 10628/12227MiB, process RSS 9734MiB, whole-system RAM 28218/31865MiB. Context 65536, 1slot, f16; max observed released slot 22583. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing authoring worker, compare each instruction with its gate, apply one narrow live-task prompt change, then append the required verification step before executing.\n\n## Strategic experiment — not applied\n\nExpose a concise task recipe diff and explicit missing-mutation progress evidence to reduce read-only authoring stalls. Validate across diverse gate/instruction mismatches without fixture-specific hints.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/facts.json): pass, fixed 8.5, 63.68minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/facts.json): pass, fixed 8.5, 78.96minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/facts.json): fail, fixed 4.5, 7.39minutes; other historical baseline; contextual only.\n\nPrevious same-scenario trials provide contextual authoring outcomes across the environment boundary. Adjacent find-versus-create and developer routing pass with one start call, but they do not test the editing capability that fails here.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nTest a generic inspect-diff-apply-verify authoring sequence across several recipe defects in a new baseline, retaining original gates and avoiding retries here.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-complex-work-craftbook-invoice-run", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-invoice-run", "suite": "complex-work", "scenario": "craftbook-invoice-run", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": false, "result": { "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T16:55:17.812Z", "finishedAt": "2026-11-10T17:49:11.131Z", "durationMs": 3233319, "success": false, "reason": "retry loop (fast-path): sniff \"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\" stuck for 12m — 3 re-writes (21 tool calls) without sniff movement. Artifact produced but never reached success.", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "failureMode": "model-stuck", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" }, "finalSniff": { "key": "craftbook-invoice-run", "score": 9, "bytes": 836, "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet", "milestones": 6 }, "failureClass": "model", "failureClassRule": "model-default" }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "model", "includedInModelAggregate": true, "failureClassRule": "model-default", "evidence": [ { "source": "result", "path": "failureClass", "value": "model" } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 3, "ruleId": "C_WATCHDOG_PARTIAL", "summary": "A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "outcome.failureMode", "value": "model-stuck" }, { "source": "facts", "path": "outcome.reason", "value": "retry loop (fast-path): sniff \"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\" stuck for 12m — 3 re-writes (21 tool calls) without sniff movement. Artifact produced but never reached success." }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 455055 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-invoice-run", "bytes": 836, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" } } ] }, "quality": { "score": 9, "ruleId": "Q_GATE_RATIO", "summary": "The latest scenario gate reached 90% of its measured signals.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": false }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 455055 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-invoice-run", "bytes": 836, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.7124079392267798 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 82 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 455055 }, { "source": "facts", "path": "outcome.durationMs", "value": 3233319 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 0 } ] } }, "composite": 5.5, "band": "needs-tuning" }, "facts": { "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "scenarioId": "craftbook-invoice-run", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 537, "peakRssMb": 15414, "peakCpuPercent": 1216.7, "firstAtMs": 0, "lastAtMs": 3215065 }, "gpu": { "available": true, "sampleCount": 537, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 537, "peakUsedMb": 31144, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 241761, "totalOutputTokens": 19746, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 147969, "todayTokensCached": 141937, "todayTokensOut": 1121, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 147969, "totalTokensCached": 141937, "totalTokensOut": 1121, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 11 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 18109, "cachedInputTokens": 18034, "outputTokens": 25, "cost": 0, "durationMs": 656290, "outputTokensPerSec": 8.600852702871718, "at": "2026-11-10T17:45:19.814Z" }, "lastUpdated": "2026-11-10T17:45:19.814Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T17:49:00.434Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3216636, "meanTokensPerSec": 8.27, "genTokensPerSec": 8.27, "genTokens": 19746, "promptTokensPerSec": 380.4, "peakRssMb": 15414, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 147969, "output": 1121 } } } }, "craftbook": { "craftbookId": "invoice-run", "tasksObserved": 9, "craftbookTasks": 1, "reachedTerminal": false, "verdict": "drove" }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 12, "activationEvents": 5, "completed": 9, "gateApprovals": 8, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "fieldnote-studio-office/1", "stepId": "scope", "activatedAt": "2026-11-10T16:55:25.330Z", "completedAt": "2026-11-10T17:02:52.979Z", "ms": 447649, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "db9738c5-acc9-47a5-96ee-9edd70eae702" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "draft", "activatedAt": "2026-11-10T17:25:14.642Z", "completedAt": "2026-11-10T17:25:14.877Z", "ms": 235, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "fieldnote-studio-office/1", "stepId": "collect", "activatedAt": "2026-11-10T17:25:14.878Z", "completedAt": "2026-11-10T17:18:49.226Z", "ms": null, "gateRejections": 0, "toolCalls": 29, "sessionIds": [ "ae5b82b5-8988-4b65-abd0-6a31bad09a82", "27bf7066-a319-40d3-9d8c-b574a11587e9" ] }, { "ref": "fieldnote-studio-office/2", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:02:53.003Z", "completedAt": "2026-11-10T17:06:46.130Z", "ms": 233127, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "6eeab097-deec-4ab8-b39e-fbc3db28c923" ] }, { "ref": "fieldnote-studio-office/3", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:02:53.123Z", "completedAt": "2026-11-10T17:10:46.285Z", "ms": 473162, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "5ec84260-6a91-4668-92ce-924426fbc615" ] }, { "ref": "fieldnote-studio-office/4", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:02:53.168Z", "completedAt": "2026-11-10T17:14:41.769Z", "ms": 708601, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "6f88b36d-e148-4a20-a33a-5102610dc1d9" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "evaluate", "activatedAt": "2026-11-10T17:18:49.227Z", "completedAt": null, "ms": null, "gateRejections": 1, "toolCalls": 15, "sessionIds": [ "7429f898-1d31-4db1-8412-bf26a2983184" ] }, { "ref": "fieldnote-studio-office/5", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:25:14.686Z", "completedAt": "2026-11-10T17:31:23.155Z", "ms": 368469, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "7c5ad6b9-de51-4453-87bc-1b1d131b444c" ] }, { "ref": "fieldnote-studio-office/6", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:25:14.743Z", "completedAt": "2026-11-10T17:37:52.599Z", "ms": 757856, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "2556b9b1-4bc9-46be-9f07-d13e7f77f903" ] }, { "ref": "fieldnote-studio-office/7", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:25:14.792Z", "completedAt": "2026-11-10T17:45:05.522Z", "ms": 1190730, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "b1b1af8e-2b12-4242-b885-9fed4aee653f" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T16:55:23.915Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T16:55:23.780Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 460405.5 }, "sessions": { "total": 12, "taskScoped": 10, "byGezel": { "amir": 6, "elsie": 1, "gonzalo": 1, "laura": 2, "naran": 1, "priya": 1 }, "perTask": { "fieldnote-studio-office/6": 1, "fieldnote-studio-office/3": 1, "fieldnote-studio-office/2": 1, "fieldnote-studio-office/4": 1, "fieldnote-studio-office/5": 1, "fieldnote-studio-office/7": 1, "fieldnote-studio-office/1": 4 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.8333333333333334, "maxMessages": 6, "maxContextFill": 0.1783447265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1783447265625 }, "fanout": { "hosts": 1, "childrenSpawned": 6, "childrenCompleted": 6, "childrenFailed": 0, "barrierHolds": 2, "barrierReleases": 2, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 9, "tasksWithReferences": 7, "items": 35, "citations": [ "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/recurring-invoice-run", "knowledge://bendyline/handboek/project-type/client-delivery", "knowledge://bendyline/handboek/craftbook/invoice-generator" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": false, "failureMode": "model-stuck", "reason": "retry loop (fast-path): sniff \"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\" stuck for 12m — 3 re-writes (21 tool calls) without sniff movement. Artifact produced but never reached success.", "durationMs": 3233319, "timeoutMs": 4538578, "budgetUsedFraction": 0.7124079392267798 }, "timing": { "startedAt": "2026-11-10T16:55:17.812Z", "finishedAt": "2026-11-10T17:49:11.131Z", "timeToFirstArtifactMs": 455055, "timeToLastArtifactWriteMs": 3218291, "timeToFirstTokenMs": 34712, "firstTurnTtftMs": 20893, "timeToFirstToolCallMs": 40741 }, "team": { "totalGezelsCreated": 9, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Office manager", "Office Administrator", "Designer", "Copywriter", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 82, "byTool": { "read_files": 3, "write_artifact": 4, "write_task_note": 2, "read_task_notes": 6, "read_file": 33, "write_file": 14, "advance_task_step": 9, "read_artifact": 8, "list_dir": 3 }, "redFlags": [] }, "artifacts": { "htmlFiles": [ { "path": "workspace/fieldnote-studio-office/invoices/2026-042.html", "finalBytes": 3916, "growth": [] }, { "path": "workspace/fieldnote-studio-office/invoices/2026-043.html", "finalBytes": 3896, "growth": [] }, { "path": "workspace/fieldnote-studio-office/invoices/2026-044.html", "finalBytes": 3880, "growth": [] } ], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 7795, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 1, "scoreMax": 10, "signals": [], "failReason": "seeded workspace input(s) have not been read yet: clients.json, ledger.json" }, { "atMs": 42682, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 686762, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 926721, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 2, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 1160350, "filePath": "craftbook-invoice-run", "bytes": 0, "score": 3, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 1374979, "filePath": "craftbook-invoice-run", "bytes": 900, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 2285745, "filePath": "craftbook-invoice-run", "bytes": 836, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 2978113, "filePath": "craftbook-invoice-run", "bytes": 876, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" }, { "atMs": 3222184, "filePath": "craftbook-invoice-run", "bytes": 836, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" } ], "latest": { "filePath": "craftbook-invoice-run", "bytes": 836, "score": 9, "scoreMax": 10, "signals": [], "failReason": "task sourced from craftbook invoice-run has not finished: its gates have not accepted the result yet" } }, "autoAnswer": { "total": 0, "byKind": { "structured": 0, "inline": 0 }, "events": [] }, "miscEvents": [ "[2026-11-10T16:55:17.816Z] [trial] id=craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml scenario=craftbook-invoice-run model=underdog-saluki-27b-iq2-mix", "[2026-11-10T16:55:17.816Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-invoice-run-ErbEuW", "[2026-11-10T16:55:17.816Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T16:55:17.816Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T16:55:17.816Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "[2026-11-10T16:55:17.819Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T16:55:17.823Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T16:55:17.828Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-invoice-run-ErbEuW", "[2026-11-10T16:55:17.828Z] [trial] throughput-scaled ceiling: 50m → 76m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T16:55:23.811Z] [trial] daemon spawned pid=45624 port=https://127.0.0.1:57736", "[2026-11-10T16:55:24.819Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T16:55:24.841Z] [trial] running scenario setup hook", "[2026-11-10T16:55:24.841Z] [trial] meester=priya", "[2026-11-10T16:55:25.501Z] [trial] scenario setup complete", "[2026-11-10T16:55:25.503Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T16:55:25.550Z] [poll] initial digests hard=2736170d2ab79483 soft=1db56660883e3c13", "[2026-11-10T16:55:25.651Z] [poll] hard-progress: 2736170d2ab79483 → a41b60dc536255f7 after 0s (workspaceProjects=3 sessions=0 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T16:55:30.771Z] [poll] hard-progress: a41b60dc536255f7 → b2ee8213ed4e9a01 after 5s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T16:56:00.555Z] [poll] hard-progress: b2ee8213ed4e9a01 → d9e9ff1221f02fad after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=23,stream=49) sniff=2/0B)", "[2026-11-10T17:02:11.042Z] [poll] hard-progress: d9e9ff1221f02fad → d9e532635767a2f1 after 370s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=1019,stream=3804) sniff=2/0B)", "[2026-11-10T17:02:53.420Z] [poll] hard-progress: d9e532635767a2f1 → f474c66fb71e55b5 after 42s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1037,stream=3857) sniff=2/0B)", "[2026-11-10T17:02:59.503Z] [poll] hard-progress: f474c66fb71e55b5 → 5192e680f370df05 after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=1037,stream=3857) sniff=2/0B)", "[2026-11-10T17:03:30.471Z] [poll] hard-progress: 5192e680f370df05 → e76765f7c8210fed after 31s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=1062,stream=3905) sniff=2/0B)", "[2026-11-10T17:03:36.279Z] [poll] hard-progress: e76765f7c8210fed → 1cd0408702f9eade after 6s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=1068,stream=3919) sniff=2/0B)", "[2026-11-10T17:06:44.680Z] [poll] hard-progress: 1cd0408702f9eade → 4d62faf2424836f8 after 188s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=1539,stream=6502) sniff=2/0B)" ] }, "metrics": { "process": { "sampleCount": 537, "peakRssMb": 15414, "peakCpuPercent": 1216.7, "firstAtMs": 0, "lastAtMs": 3215065 }, "gpu": { "available": true, "sampleCount": 537, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 537, "peakUsedMb": 31144, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 241761, "totalOutputTokens": 19746, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 11, "todayTokensIn": 147969, "todayTokensCached": 141937, "todayTokensOut": 1121, "todayCost": 0, "totalTurns": 11, "totalTokensIn": 147969, "totalTokensCached": 141937, "totalTokensOut": 1121, "totalCost": 0, "medianOutputTokensPerSec": 8.3, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.3, "turns": 11 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 18109, "cachedInputTokens": 18034, "outputTokens": 25, "cost": 0, "durationMs": 656290, "outputTokensPerSec": 8.600852702871718, "at": "2026-11-10T17:45:19.814Z" }, "lastUpdated": "2026-11-10T17:45:19.814Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T17:49:00.434Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3216636, "meanTokensPerSec": 8.27, "genTokensPerSec": 8.27, "genTokens": 19746, "promptTokensPerSec": 380.4, "peakRssMb": 15414, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 147969, "output": 1121 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T16:55:17.812Z", "finishedAt": "2026-11-10T17:49:11.131Z", "timeToFirstArtifactMs": 455055, "timeToLastArtifactWriteMs": 3218291, "timeToFirstTokenMs": 34712, "firstTurnTtftMs": 20893, "timeToFirstToolCallMs": 40741 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 12, "activationEvents": 5, "completed": 9, "gateApprovals": 8, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "fieldnote-studio-office/1", "stepId": "scope", "activatedAt": "2026-11-10T16:55:25.330Z", "completedAt": "2026-11-10T17:02:52.979Z", "ms": 447649, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "db9738c5-acc9-47a5-96ee-9edd70eae702" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "draft", "activatedAt": "2026-11-10T17:25:14.642Z", "completedAt": "2026-11-10T17:25:14.877Z", "ms": 235, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "fieldnote-studio-office/1", "stepId": "collect", "activatedAt": "2026-11-10T17:25:14.878Z", "completedAt": "2026-11-10T17:18:49.226Z", "ms": null, "gateRejections": 0, "toolCalls": 29, "sessionIds": [ "ae5b82b5-8988-4b65-abd0-6a31bad09a82", "27bf7066-a319-40d3-9d8c-b574a11587e9" ] }, { "ref": "fieldnote-studio-office/2", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:02:53.003Z", "completedAt": "2026-11-10T17:06:46.130Z", "ms": 233127, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "6eeab097-deec-4ab8-b39e-fbc3db28c923" ] }, { "ref": "fieldnote-studio-office/3", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:02:53.123Z", "completedAt": "2026-11-10T17:10:46.285Z", "ms": 473162, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "5ec84260-6a91-4668-92ce-924426fbc615" ] }, { "ref": "fieldnote-studio-office/4", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:02:53.168Z", "completedAt": "2026-11-10T17:14:41.769Z", "ms": 708601, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "6f88b36d-e148-4a20-a33a-5102610dc1d9" ] }, { "ref": "fieldnote-studio-office/1", "stepId": "evaluate", "activatedAt": "2026-11-10T17:18:49.227Z", "completedAt": null, "ms": null, "gateRejections": 1, "toolCalls": 15, "sessionIds": [ "7429f898-1d31-4db1-8412-bf26a2983184" ] }, { "ref": "fieldnote-studio-office/5", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:25:14.686Z", "completedAt": "2026-11-10T17:31:23.155Z", "ms": 368469, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "7c5ad6b9-de51-4453-87bc-1b1d131b444c" ] }, { "ref": "fieldnote-studio-office/6", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:25:14.743Z", "completedAt": "2026-11-10T17:37:52.599Z", "ms": 757856, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "2556b9b1-4bc9-46be-9f07-d13e7f77f903" ] }, { "ref": "fieldnote-studio-office/7", "stepId": "draft-invoice", "activatedAt": "2026-11-10T17:25:14.792Z", "completedAt": "2026-11-10T17:45:05.522Z", "ms": 1190730, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "b1b1af8e-2b12-4242-b885-9fed4aee653f" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T16:55:23.915Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T16:55:23.780Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 460405.5 }, "sessions": { "total": 12, "taskScoped": 10, "byGezel": { "amir": 6, "elsie": 1, "gonzalo": 1, "laura": 2, "naran": 1, "priya": 1 }, "perTask": { "fieldnote-studio-office/6": 1, "fieldnote-studio-office/3": 1, "fieldnote-studio-office/2": 1, "fieldnote-studio-office/4": 1, "fieldnote-studio-office/5": 1, "fieldnote-studio-office/7": 1, "fieldnote-studio-office/1": 4 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.8333333333333334, "maxMessages": 6, "maxContextFill": 0.1783447265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1783447265625 }, "fanout": { "hosts": 1, "childrenSpawned": 6, "childrenCompleted": 6, "childrenFailed": 0, "barrierHolds": 2, "barrierReleases": 2, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-complex-work-craftbook-invoice-run", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-invoice-run", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "scenarioId": "craftbook-invoice-run", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "scored", "provisional": false, "reviewedAt": "2026-11-10T20:30:54.663Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/report.md", "artifactBytes": 842, "artifactSha256": "279e90551f2de96f799902c411c0f98fb4611664cf557a8441082fd3d19001af", "axisSource": "packages/catalog/node_modules/@bendyline/gilde/data/craftbook-templates/in/invoice-run/versions/1.1.9/test.json", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ], "scoreAxes": { "numeric integrity": 8, "numbering continuity": 8, "professionalism": 6, "completeness": 4, "print-polish": 7 }, "meanScore": 6.6, "justification": "Three invoices correctly total5415, continue042–044 and use terms-correct due dates; the idle client is skipped. Clear tables and identical print CSS support paper use, but no render was checked. Books remain at041/uninvoiced. The all-PASS review is rejected after the write tool appends Sources lines, triggering repeat fanout and a stall.", "reviewBasis": "Read report, all three invoices, source books and working artifacts with20KiB per-file cap; compared CSS hashes and recorded write/gate receipts. Print rating is source-based; no browser or paper rendering was performed.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/report.md", "status": "scored", "bytes": 842, "sha256": "279e90551f2de96f799902c411c0f98fb4611664cf557a8441082fd3d19001af", "reviewedBytes": 842, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-042.html", "status": "scored", "bytes": 3916, "sha256": "c9e5993975521e99711b99ada34cb84b103b7f75cc769e5d1f993aaed52e6712", "reviewedBytes": 3916, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-043.html", "status": "scored", "bytes": 3896, "sha256": "affe16ddab8f14c8ffe13f8c5cd8569e99d0f5af95acb19dada69dbe3c2f1d44", "reviewedBytes": 3896, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-044.html", "status": "scored", "bytes": 3880, "sha256": "7eba50a1c71c5403ff0e279780f0cacde72b3a00cb3bd5ee6b4dcf9567b1590a", "reviewedBytes": 3880, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/ledger.json", "status": "notApplicable", "bytes": 878, "sha256": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073", "reviewedBytes": 878, "truncated": false, "role": "sourceBooks" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/scope.md", "status": "notApplicable", "bytes": 1781, "sha256": "b818836fb6ee8dda8d899b7aeb0fa7287410b9a96e9f536690161510ea1188d8", "reviewedBytes": 1781, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/billables.json", "status": "notApplicable", "bytes": 539, "sha256": "4513368faac4a8b19e00e6ce11e00d9036527597761966314db708265bd735ee", "reviewedBytes": 539, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/verdict.md", "status": "notApplicable", "bytes": 870, "sha256": "1727e7dda22f9942d7bc6d1b23e0e87483aec0a37726fc1233333505103b11f8", "reviewedBytes": 870, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T16:55:29.788Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T16:55:32.952Z INFO [chat] [llama-server] launch {\"pid\":11508,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T17:49:10.385Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-11508-1794332950385\",\"pid\":11508,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":3217434,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 22837, "peakGpuMiB": 10644, "peakProcessRssMiB": 15414, "peakSystemRamMiB": 31144, "decodeTokensPerSec": 8.27, "prefillTokensPerSec": 380.4, "generatedTokens": 19746, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 0, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.1783447265625 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:07:16.974Z", "reviewId": "123", "cellId": "r3-saluki-cuda-complex-work-craftbook-invoice-run", "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "scenarioId": "craftbook-invoice-run", "status": "scored", "axes": [ { "name": "numeric integrity", "description": "Every amount traces to a tracked ledger entry; nothing estimated or invented." }, { "name": "numbering continuity", "description": "Invoice numbers continue 2026-041 with no skip and no repeat." }, { "name": "professionalism", "description": "The invoices look worth paying — clear line items, terms-correct due dates, print-ready pages." }, { "name": "completeness", "description": "One invoice per billable client, the idle client skipped on purpose, and the books updated." }, { "name": "print-polish", "description": "Each invoice is styled for paper — an @page rule, honest margins, a professional face." } ], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-invoice-run", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "8f397356a7ec750cc169bf4607f024caad9ebc5d9cf1493b3133193d4842895d", "evidenceSha256": "6c2e631e86feaf898e81d0599b7a0de4789481fbabc20e7849be904ef989e7fe", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/report.md", "bytes": 842, "sha256": "279e90551f2de96f799902c411c0f98fb4611664cf557a8441082fd3d19001af", "shownBytes": 842, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-042.html", "bytes": 3916, "sha256": "c9e5993975521e99711b99ada34cb84b103b7f75cc769e5d1f993aaed52e6712", "shownBytes": 3916, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-043.html", "bytes": 3896, "sha256": "affe16ddab8f14c8ffe13f8c5cd8569e99d0f5af95acb19dada69dbe3c2f1d44", "shownBytes": 3896, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-044.html", "bytes": 3880, "sha256": "7eba50a1c71c5403ff0e279780f0cacde72b3a00cb3bd5ee6b4dcf9567b1590a", "shownBytes": 3880, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/ledger.json", "bytes": 878, "sha256": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073", "shownBytes": 878, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/scope.md", "bytes": 1781, "sha256": "b818836fb6ee8dda8d899b7aeb0fa7287410b9a96e9f536690161510ea1188d8", "shownBytes": 1781, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/billables.json", "bytes": 539, "sha256": "4513368faac4a8b19e00e6ce11e00d9036527597761966314db708265bd735ee", "shownBytes": 539, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/verdict.md", "bytes": 870, "sha256": "1727e7dda22f9942d7bc6d1b23e0e87483aec0a37726fc1233333505103b11f8", "shownBytes": 870, "truncated": false } ], "contextFiles": [], "scoreAxes": { "numeric integrity": 9, "numbering continuity": 10, "professionalism": 7, "completeness": 5, "print-polish": 8 }, "justification": "Amounts (1,840/2,600/975; total 5,415) match uninvoiced ledger entries, and the 1,200 already-invoiced entry was excluded. Numbers 042–044 follow 041. Due dates are correct (net 30 → 12-10, net 14 → 11-24). Three invoices, Ondaatje skipped and named. But ledger.json still shows invoiced:false and lastInvoiceNumber 2026-041, so the books weren't updated.", "limitations": "Source-only review, with no rendering. The ledger snapshot may predate a later update step. Invoice line items are single-row and omit the work dates. The ledger entries are dated July while the run is November.", "meanScore": 7.8, "durationMs": 17814, "costUsd": 0.0543032, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 11615, "cache_read_input_tokens": 892, "output_tokens": 775, "output_tokens_details": { "thinking_tokens": 388 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 11615, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 775, "cache_read_input_tokens": 892, "cache_creation_input_tokens": 11615, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 11615 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 775, "cacheReadInputTokens": 892, "cacheCreationInputTokens": 11615, "webSearchRequests": 0, "costUSD": 0.0543032, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 388, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 11615, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-invoice-run / underdog-saluki-27b-iq2-mix\n\n**Composite: 5.5 / 10** (band: needs-tuning)\n**Outcome: model-stuck** (duration: 53m 53s; budget used: 71.2%)\n**Model aggregate eligibility: included (model).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 3 / 10 | A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\\\" stuck for 12m — 3 re-writes (21 tool calls) without sniff movement. Artifact produced b...`; `facts.timing.timeToFirstArtifactMs: 455055`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":836,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...` |\n| Output quality (25%) | 9 / 10 | The latest scenario gate reached 90% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 455055`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":836,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.7124079392267798`; `facts.toolUse.totalToolCalls: 82`; `facts.timing.timeToFirstArtifactMs: 455055`; `facts.outcome.durationMs: 3233319` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0` |\n\nComposite = 0.4 * 3 + 0.25 * 9 + 0.2 * 2.5 + 0.15 * 10 = **5.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15414 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31144 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.27 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 241761 + 19746 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 12 → 9 | `continuity.steps` |\n| Gate rejections / redrives | 1 / 0 | `continuity.steps` |\n| Median step wall-clock | 460s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 10 / 0.83 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 0 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 18% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 6 / 6 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 2 / 2 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 7 of 9 / 35 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_WATCHDOG_PARTIAL\n\nA watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator.\n\n- `facts.outcome.success: false`\n- `facts.outcome.failureMode: \"model-stuck\"`\n- `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\\\" stuck for 12m — 3 re-writes (21 tool calls) without sniff movement. Artifact produced b...`\n- `facts.timing.timeToFirstArtifactMs: 455055`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":836,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...`\n\n### Output quality - Q_GATE_RATIO\n\nThe latest scenario gate reached 90% of its measured signals.\n\n- `facts.outcome.success: false`\n- `facts.timing.timeToFirstArtifactMs: 455055`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":836,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.7124079392267798`\n- `facts.toolUse.totalToolCalls: 82`\n- `facts.timing.timeToFirstArtifactMs: 455055`\n- `facts.outcome.durationMs: 3233319`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 0`\n\n## Evidence-led observations\n\n- Completion: A watchdog ended the trial after at least half of a measured gate, or after observable progress with an unavailable denominator. `facts.outcome.success: false`; `facts.outcome.failureMode: \"model-stuck\"`; `facts.outcome.reason: \"retry loop (fast-path): sniff \\\"craftbook-invoice-run:9:targetnone:fraw98pc:rp0:rf0:m6\\\" stuck for 12m — 3 re-writes (21 tool calls) without sniff movement. Artifact produced b...`; `facts.timing.timeToFirstArtifactMs: 455055`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":836,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...`\n- Quality: The latest scenario gate reached 90% of its measured signals. `facts.outcome.success: false`; `facts.timing.timeToFirstArtifactMs: 455055`; `facts.sniff.latest: {\"filePath\":\"craftbook-invoice-run\",\"bytes\":836,\"score\":9,\"scoreMax\":10,\"signals\":[],\"failReason\":\"task sourced from craftbook invoice-run has not finished: its gates have not a...`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.7124079392267798`; `facts.toolUse.totalToolCalls: 82`; `facts.timing.timeToFirstArtifactMs: 455055`; `facts.outcome.durationMs: 3233319`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 0`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **5.5/10**, eligible task failure; unchanged. Advisory **6.6/10**. Three invoices correctly total5415, continue042–044 and use terms-correct due dates; the idle client is skipped. Clear tables and identical print CSS support paper use, but no render was checked. Books remain at041/uninvoiced. The all-PASS review is rejected after the write tool appends Sources lines, triggering repeat fanout and a stall.\n\nExact declared axes: numeric integrity=8; numbering continuity=8; professionalism=6; completeness=4; print-polish=7.\n\n## Positives\n\nAll three amounts trace to the ledger; the already billed1200 is excluded. Source inspection confirms contiguous numbers and net14/net30 dates from November10. All three HTML styles hash identically and specify letter paper,28mm/24mm margins, table break avoidance and print rules. Six child tasks finish across two passes (facts.continuity.fanout); all invoice files and summary exist.\n\n## Negatives and diagnosis\n\nThe authored review ends each criterion PASS, yet the runtime sends it back to draft. The saved verdict includes an unrequested Sources heading/footnotes appended by the write tool; packages/service/src/tasks/review-verdict.ts rejects non-indented non-decision lines. The first offending line is the Sources heading, consistent with the recorded notContains rejection, not evidence of a literal FAIL. The earlier grounding guard splits comma-formatted amounts into unsupported numeric fragments. After repeat fanout, two stale collect advances are rejected and the watchdog ends the trial. The ledger remains lastInvoiceNumber041 with all three entries uninvoiced; the template has no explicit books-update step despite the declared completeness axis requiring one. Report wording says went out without a send receipt. Keep the fixed model-classified stall unchanged while recording these framework/content contributions.\n\n## Performance and fit\n\nDecode 8.27tok/s, prefill 380.4tok/s; 19746generated tokens. Peak GPU 10644/12227MiB, process RSS 15414MiB, whole-system RAM 31144/31865MiB. Context 65536, 1slot, f16; max observed released slot 22837. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing invoice template, distinguish drafted from sent and define when the ledger is updated after acceptance; do not imply delivery or change books during a rejected pass. Add remittance/currency details only when supplied by the source books.\n\n## Strategic experiment — not applied\n\nExperiment with compatible provenance and structured review contracts in packages/service/src/tasks/review-verdict.ts and the artifact grounding/citation writer: metadata must not invalidate an otherwise valid verdict. Preserve rejection of real FAIL/ambiguous criteria. Improve decimal/grouped-number evidence matching and expose actual gate detail rather than only the notContains label.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/facts.json): pass, fixed 8.5, 28.44minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/facts.json): fail, fixed 5.5, 58.74minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/facts.json): pass, fixed 8.5, 3.21minutes; other historical baseline; contextual only.\n\nFanout creates all requested invoices twice, demonstrating executable task decomposition but poor recovery from a review/provenance contract conflict. Linear authoring completes actual data cleanup, while edit-midtask never mutates its recipe. Earlier invoice outcomes and unrelated source-grounded report failures are contextual across the explicit environment boundary.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nIn a fresh baseline, test citation-bearing all-PASS and real-FAIL reviews across invoice, code-review and release-writing workflows; verify routing and books-state invariants without weakening their gates.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-complex-work-craftbook-author-linear", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-linear", "suite": "complex-work", "scenario": "craftbook-author-linear", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T17:52:16.270Z", "finishedAt": "2026-11-10T18:49:11.540Z", "durationMs": 3415270, "success": true, "reason": "authored a gated 3-step craftbook, invoked it, ran the task to completion, and all three outputs pass their checks", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1201605 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-linear", "bytes": 7865, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.4703111175374997 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 65 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1201605 }, { "source": "facts", "path": "outcome.durationMs", "value": 3415270 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "scenarioId": "craftbook-author-linear", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 568, "peakRssMb": 15745, "peakCpuPercent": 1238.5, "firstAtMs": 0, "lastAtMs": 3400302 }, "gpu": { "available": true, "sampleCount": 568, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 568, "peakUsedMb": 31290, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 158378, "totalOutputTokens": 22345, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 145729, "todayTokensCached": 144121, "todayTokensOut": 1642, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 145729, "totalTokensCached": 144121, "totalTokensOut": 1642, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 18120, "cachedInputTokens": 17856, "outputTokens": 100, "cost": 0, "durationMs": 1282486, "outputTokensPerSec": 8.418981588962868, "at": "2026-11-10T18:49:03.109Z" }, "lastUpdated": "2026-11-10T18:49:03.109Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T18:49:03.866Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3401827, "meanTokensPerSec": 8.35, "genTokensPerSec": 8.35, "genTokens": 22345, "promptTokensPerSec": 336.57, "peakRssMb": 15745, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 145729, "output": 1642 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "order-intake-cleanup/1", "stepId": "inspect", "activatedAt": "2026-11-10T18:04:07.618Z", "completedAt": "2026-11-10T18:14:59.345Z", "ms": 651727, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "c3f742b1-239c-44f1-99ff-208e79aee483" ] }, { "ref": "order-intake-cleanup/1", "stepId": "clean", "activatedAt": "2026-11-10T18:14:59.347Z", "completedAt": "2026-11-10T18:26:33.675Z", "ms": 694328, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "8e1ab77f-e7f9-419c-b137-a807e1e96fd8" ] }, { "ref": "order-intake-cleanup/1", "stepId": "verify", "activatedAt": "2026-11-10T18:26:33.677Z", "completedAt": "2026-11-10T18:49:03.004Z", "ms": 1349327, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "43c75f62-6067-41c4-8393-6dca735fd678" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T17:52:22.073Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T17:52:21.976Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 694328 }, "sessions": { "total": 5, "taskScoped": 3, "byGezel": { "bohdan": 2, "njeri": 1, "reza": 1, "salvador": 1 }, "perTask": { "order-intake-cleanup/1": 3 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6, "maxMessages": 10, "maxContextFill": 0.3297882080078125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3297882080078125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbook/dataset-clean", "knowledge://bendyline/handboek/craftbooks-index", "knowledge://bendyline/handboek/craftbook/reorder-run", "knowledge://bendyline/handboek/craftbook/scrape-to-structured" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "authored a gated 3-step craftbook, invoked it, ran the task to completion, and all three outputs pass their checks", "durationMs": 3415270, "timeoutMs": 7261725, "budgetUsedFraction": 0.4703111175374997 }, "timing": { "startedAt": "2026-11-10T17:52:16.270Z", "finishedAt": "2026-11-10T18:49:11.540Z", "timeToFirstArtifactMs": 1201605, "timeToLastArtifactWriteMs": 3406840, "timeToFirstTokenMs": 47361, "firstTurnTtftMs": 37091, "timeToFirstToolCallMs": 64636 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Data-Analyst", "Developer", "Reviewer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 65, "byTool": { "list_craftbooks": 1, "craftbook_read": 2, "craftbook_write": 1, "invoke_craftbook": 1, "list_tasks": 3, "read_task_notes": 9, "stat": 13, "get_task": 5, "read_file": 12, "write_task_note": 4, "list_project_gezels": 1, "message_gezel": 1, "write_file": 4, "advance_task_step": 3, "derive_file": 2, "validate": 2, "list_dir": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 5 }, "sniff": { "progression": [ { "atMs": 7283, "filePath": "craftbook-author-linear", "bytes": 0, "score": 3, "scoreMax": 7, "signals": [], "failReason": "no reusable authored craftbook with at least 3 steps exists yet — author \"CSV Order Cleanup\" (inspect, clean, verify) as a craftbook template, not as ad-hoc chat work" }, { "atMs": 683213, "filePath": "craftbook-author-linear", "bytes": 28, "score": 3, "scoreMax": 7, "signals": [], "failReason": "no task has been created from craftbook \"csv-order-cleanup\" yet — invoke the craftbook on this project" }, { "atMs": 716102, "filePath": "craftbook-author-linear", "bytes": 28, "score": 3, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 1206794, "filePath": "craftbook-author-linear", "bytes": 1338, "score": 4, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 1785791, "filePath": "craftbook-author-linear", "bytes": 5987, "score": 5, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3216358, "filePath": "craftbook-author-linear", "bytes": 7865, "score": 6, "scoreMax": 7, "signals": [], "failReason": "task order-intake-cleanup/1 (from craftbook \"csv-order-cleanup\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3407148, "filePath": "craftbook-author-linear", "bytes": 7865, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-author-linear", "bytes": 7865, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 3407142, "kind": "structured", "gezel": "reza", "question": "**CSV Order Cleanup — data/orders.csv** is finished. - **`out/report.md`** - `n…", "chose": "write-in (workspace-fixture)" } ] }, "miscEvents": [ "[2026-11-10T17:52:16.273Z] [trial] id=craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm scenario=craftbook-author-linear model=underdog-saluki-27b-iq2-mix", "[2026-11-10T17:52:16.273Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-linear-psV2zX", "[2026-11-10T17:52:16.273Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "[2026-11-10T17:52:16.273Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T17:52:16.275Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T17:52:16.273Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T17:52:16.279Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T17:52:16.284Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-linear-psV2zX", "[2026-11-10T17:52:16.285Z] [trial] throughput-scaled ceiling: 80m → 121m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T17:52:22.036Z] [trial] daemon spawned pid=15492 port=https://127.0.0.1:49283", "[2026-11-10T17:52:22.913Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T17:52:22.933Z] [trial] running scenario setup hook", "[2026-11-10T17:52:22.933Z] [trial] meester=jasper", "[2026-11-10T17:52:23.301Z] [trial] scenario setup complete", "[2026-11-10T17:52:23.303Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T17:52:23.382Z] [poll] initial digests hard=7c005ea70c7917d2 soft=5f442a350ccc94c6", "[2026-11-10T17:52:23.713Z] [poll] hard-progress: 7c005ea70c7917d2 → 80c73fa6a844fda7 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=3/0B)", "[2026-11-10T17:53:22.930Z] [poll] hard-progress: 80c73fa6a844fda7 → 596fd8ba6344c8ec after 59s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=64,stream=142) sniff=3/0B)", "[2026-11-10T17:53:46.901Z] [poll] hard-progress: 596fd8ba6344c8ec → ca88e843809a8dce after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=87,stream=199) sniff=3/0B)", "[2026-11-10T18:03:39.603Z] [poll] hard-progress: ca88e843809a8dce → eea4233c4d3d04b1 after 593s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=1669,stream=5992) sniff=3/28B)", "[2026-11-10T18:04:12.441Z] [poll] hard-progress: eea4233c4d3d04b1 → 9c463f68fcb967d5 after 33s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=1731,stream=6287) sniff=3/28B)", "[2026-11-10T18:04:24.592Z] [poll] hard-progress: 9c463f68fcb967d5 → 77fb2f0b72373469 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=1755,stream=6358) sniff=3/28B)", "[2026-11-10T18:04:36.875Z] [poll] hard-progress: 77fb2f0b72373469 → b3603272d411c603 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=1780,stream=6434) sniff=3/28B)", "[2026-11-10T18:04:55.080Z] [poll] hard-progress: b3603272d411c603 → dc707e47f7a4b7b4 after 18s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=1821,stream=6558) sniff=3/28B)", "[2026-11-10T18:05:07.257Z] [poll] hard-progress: dc707e47f7a4b7b4 → 3c5b9d8bc28f64dd after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=8,slot=1845,stream=6633) sniff=3/28B)" ] }, "metrics": { "process": { "sampleCount": 568, "peakRssMb": 15745, "peakCpuPercent": 1238.5, "firstAtMs": 0, "lastAtMs": 3400302 }, "gpu": { "available": true, "sampleCount": 568, "peakUtilPercent": 100, "peakMemUsedMb": 10644, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 568, "peakUsedMb": 31290, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 158378, "totalOutputTokens": 22345, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 145729, "todayTokensCached": 144121, "todayTokensOut": 1642, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 145729, "totalTokensCached": 144121, "totalTokensOut": 1642, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 18120, "cachedInputTokens": 17856, "outputTokens": 100, "cost": 0, "durationMs": 1282486, "outputTokensPerSec": 8.418981588962868, "at": "2026-11-10T18:49:03.109Z" }, "lastUpdated": "2026-11-10T18:49:03.109Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T18:49:03.866Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3401827, "meanTokensPerSec": 8.35, "genTokensPerSec": 8.35, "genTokens": 22345, "promptTokensPerSec": 336.57, "peakRssMb": 15745, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 145729, "output": 1642 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T17:52:16.270Z", "finishedAt": "2026-11-10T18:49:11.540Z", "timeToFirstArtifactMs": 1201605, "timeToLastArtifactWriteMs": 3406840, "timeToFirstTokenMs": 47361, "firstTurnTtftMs": 37091, "timeToFirstToolCallMs": 64636 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 5, "activationEvents": 2, "completed": 3, "gateApprovals": 3, "gateRejections": 0, "redrives": 0, "perStep": [ { "ref": "order-intake-cleanup/1", "stepId": "inspect", "activatedAt": "2026-11-10T18:04:07.618Z", "completedAt": "2026-11-10T18:14:59.345Z", "ms": 651727, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "c3f742b1-239c-44f1-99ff-208e79aee483" ] }, { "ref": "order-intake-cleanup/1", "stepId": "clean", "activatedAt": "2026-11-10T18:14:59.347Z", "completedAt": "2026-11-10T18:26:33.675Z", "ms": 694328, "gateRejections": 0, "toolCalls": 7, "sessionIds": [ "8e1ab77f-e7f9-419c-b137-a807e1e96fd8" ] }, { "ref": "order-intake-cleanup/1", "stepId": "verify", "activatedAt": "2026-11-10T18:26:33.677Z", "completedAt": "2026-11-10T18:49:03.004Z", "ms": 1349327, "gateRejections": 0, "toolCalls": 11, "sessionIds": [ "43c75f62-6067-41c4-8393-6dca735fd678" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T17:52:22.073Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T17:52:21.976Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 694328 }, "sessions": { "total": 5, "taskScoped": 3, "byGezel": { "bohdan": 2, "njeri": 1, "reza": 1, "salvador": 1 }, "perTask": { "order-intake-cleanup/1": 3 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6, "maxMessages": 10, "maxContextFill": 0.3297882080078125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3297882080078125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-complex-work-craftbook-author-linear", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-linear", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "scenarioId": "craftbook-author-linear", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-10T20:31:02.476Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/out/report.md", "artifactBytes": 1908, "artifactSha256": "46477033a36d64c6fc5be3caf52f40b60241673eb788ff0dd66afd5e349f7851", "axisSource": "evals/src/craftbooks/authoring/author-linear.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "The model authors and executes a three-step inspect/clean/verify recipe. All29output records independently match the30-row CSV after fixing qty two→2 and dropping the impossible date. Reports identify both anomalies accurately. Gates check nonempty files, so semantic success rests on the inspected data rather than gate strength.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/out/report.md", "status": "notApplicable", "bytes": 1908, "sha256": "46477033a36d64c6fc5be3caf52f40b60241673eb788ff0dd66afd5e349f7851", "reviewedBytes": 1908, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/notes/anomalies.md", "status": "notApplicable", "bytes": 1328, "sha256": "70e012fd3f6d32ccc48126aabb5932442a1a8fe1ddeab437d66f4d97ec74f601", "reviewedBytes": 1328, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/out/orders.json", "status": "notApplicable", "bytes": 4649, "sha256": "1b249c9142fa797f5ee04c47e946499196fa284d603d1a80b87707558f233410", "reviewedBytes": 4649, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/data/orders.csv", "status": "notApplicable", "bytes": 1546, "sha256": "c88785ed78c4d8b14484d8d0f04516c9c93c99045b2e90b60b19f96ee17d7bb2", "reviewedBytes": 1546, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T17:52:24.652Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T17:52:27.883Z INFO [chat] [llama-server] launch {\"pid\":25096,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T18:49:10.768Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-25096-1794336550768\",\"pid\":25096,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":3402885,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 43910, "peakGpuMiB": 10644, "peakProcessRssMiB": 15745, "peakSystemRamMiB": 31290, "decodeTokensPerSec": 8.35, "prefillTokensPerSec": 336.57, "generatedTokens": 22345, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.3297882080078125 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:07:15.790Z", "reviewId": "124", "cellId": "r3-saluki-cuda-complex-work-craftbook-author-linear", "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "scenarioId": "craftbook-author-linear", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-linear", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "5ee224cc81bf330e050cc3ddf64cdeaa6ff3f3b6d5e91682189362031a3b6ae6", "evidenceSha256": "bcd95332ff3c48ec96b0d8da00d2b862bc6becd7c6933c7d003515436c896999", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/out/report.md", "bytes": 1908, "sha256": "46477033a36d64c6fc5be3caf52f40b60241673eb788ff0dd66afd5e349f7851", "shownBytes": 1908, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/notes/anomalies.md", "bytes": 1328, "sha256": "70e012fd3f6d32ccc48126aabb5932442a1a8fe1ddeab437d66f4d97ec74f601", "shownBytes": 1328, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/out/orders.json", "bytes": 4649, "sha256": "1b249c9142fa797f5ee04c47e946499196fa284d603d1a80b87707558f233410", "shownBytes": 4649, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/data/orders.csv", "bytes": 1546, "sha256": "c88785ed78c4d8b14484d8d0f04516c9c93c99045b2e90b60b19f96ee17d7bb2", "shownBytes": 1546, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes declared. Output files are consistent with the CSV: anomalies.md found the two bad rows (ORD-1013 qty \"two\", ORD-1022 date 2026-13-45); orders.json has 29 valid records (1013 fixed, 1022 dropped); report.md accurately reports PASS.", "limitations": "The craftbook document, craftbook_write/invoke_craftbook calls, step deliverable declarations, and ordering (authored before delegation) are not in the evidence. Only output files were reviewed, so template authoring and task completion are unverified.", "meanScore": null, "durationMs": 14557, "costUsd": 0.035862, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 8182, "cache_read_input_tokens": 700, "output_tokens": 306, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 8182, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 306, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 8182, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 8182 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 306, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 8182, "webSearchRequests": 0, "costUSD": 0.035862, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 8182, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-linear / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 56m 55s; budget used: 47.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1201605`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7865,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.4703111175374997`; `facts.toolUse.totalToolCalls: 65`; `facts.timing.timeToFirstArtifactMs: 1201605`; `facts.outcome.durationMs: 3415270` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 15745 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10644 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 31290 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.35 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 158378 + 22345 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 0 / 0 | `continuity.steps` |\n| Median step wall-clock | 694s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 3 / 0.60 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 1 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 33% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1201605`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7865,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.4703111175374997`\n- `facts.toolUse.totalToolCalls: 65`\n- `facts.timing.timeToFirstArtifactMs: 1201605`\n- `facts.outcome.durationMs: 3415270`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1201605`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-linear\",\"bytes\":7865,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.4703111175374997`; `facts.toolUse.totalToolCalls: 65`; `facts.timing.timeToFirstArtifactMs: 1201605`; `facts.outcome.durationMs: 3415270`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.5/10**, pass; unchanged. Advisory **notApplicable**. The model authors and executes a three-step inspect/clean/verify recipe. All29output records independently match the30-row CSV after fixing qty two→2 and dropping the impossible date. Reports identify both anomalies accurately. Gates check nonempty files, so semantic success rests on the inspected data rather than gate strength.\n\n\n\n## Positives\n\nThe craftbook_write receipt creates three gated steps with data-analyst, developer and reviewer roles; the task finishes. Independent source-to-output comparison verifies all six fields on29records, numeric conversions, the ORD-1013 quantity repair and ORD-1022 exclusion. Notes correctly identify original row numbers and preserve the remaining records.\n\n## Negatives and diagnosis\n\nThe recipe hard-codes a minimum20records despite claiming reuse for any order export; all three gates are size/nonempty checks, not semantic validators. An initial derive_file helper uses require under ESM and fails, then an import-based repair works. Repeated task/status polling adds delay. Grounding catches one wrong customer but also rejects derived row counts. These are efficiency/generalization limits, not a failure of the delivered fixture cleanup.\n\n## Performance and fit\n\nDecode 8.35tok/s, prefill 336.57tok/s; 22345generated tokens. Peak GPU 10644/12227MiB, process RSS 15745MiB, whole-system RAM 31290/31865MiB. Context 65536, 1slot, f16; max observed released slot 43910. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nFor the existing authoring worker, express record-retention constraints as source-dependent parameters rather than copying an absolute threshold into a reusable recipe. Use the documented ESM execution shape for derive_file and progress events instead of repeated status polling.\n\n## Strategic experiment — not applied\n\nUse schema/date/row-reconciliation gates in authored data-cleanup recipes and attach deterministic transformation receipts to reports. Validate empty, small, quoted-field and duplicate-ID inputs without assuming this fixture size.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/facts.json): pass, fixed 8.5, 66.99minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/facts.json): pass, fixed 8.5, 41.76minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/facts.json): fail, fixed 5.4, 17.33minutes; other historical baseline; contextual only.\n\nThis is a genuine end-to-end authoring success, unlike route-only selection or edit-midtask inspection stalls. Data-wrangle also reconciles source records correctly; records-intake produces no final CSV and AB analysis never closes. Thus data handling competence is uneven across workflow/tool shapes.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nEvaluate reusable cleanup recipes on varied CSV schemas and sizes plus an unrelated document workflow, measuring semantic accuracy and time spent polling in a fresh baseline.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-complex-work-craftbook-route-multi", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-route-multi", "suite": "complex-work", "scenario": "craftbook-route-multi", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T18:52:14.367Z", "finishedAt": "2026-11-10T19:51:07.739Z", "durationMs": 3533372, "success": true, "reason": "invoked the bundled press-release recipe, ran the task to completion, and the release carries a dateline, the attributed approved quote, the boilerplate, a contact, and the approved facts rather than the stale draft", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 507896 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-route-multi", "bytes": 1147, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.4865747463584754 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 47 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 507896 }, { "source": "facts", "path": "outcome.durationMs", "value": 3533372 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "scenarioId": "craftbook-route-multi", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 589, "peakRssMb": 14143, "peakCpuPercent": 1372.2, "firstAtMs": 1, "lastAtMs": 3521164 }, "gpu": { "available": true, "sampleCount": 589, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 589, "peakUsedMb": 30766, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 151149, "totalOutputTokens": 24430, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 148048, "todayTokensCached": 144983, "todayTokensOut": 4072, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 148048, "totalTokensCached": 144983, "totalTokensOut": 4072, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 9 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9666, "cachedInputTokens": 9282, "outputTokens": 177, "cost": 0, "durationMs": 335941, "outputTokensPerSec": 8.406529848482808, "at": "2026-11-10T19:51:02.232Z" }, "lastUpdated": "2026-11-10T19:51:02.232Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T19:51:02.742Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3522535, "meanTokensPerSec": 8.4, "genTokensPerSec": 8.4, "genTokens": 24430, "promptTokensPerSec": 334.88, "peakRssMb": 14143, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 148048, "output": 4072 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 5, "completed": 5, "gateApprovals": 5, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "fieldstone-launch-comms/1", "stepId": "scope", "activatedAt": "2026-11-10T18:54:15.866Z", "completedAt": "2026-11-10T19:00:42.459Z", "ms": 386593, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "df3d4cb8-4d73-47c2-9b55-5bafdd4762f2" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "draft", "activatedAt": "2026-11-10T19:07:10.326Z", "completedAt": "2026-11-10T19:18:14.902Z", "ms": 664576, "gateRejections": 1, "toolCalls": 6, "sessionIds": [ "945af339-4090-4423-a2f2-a918f860fa8c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "review", "activatedAt": "2026-11-10T19:18:14.903Z", "completedAt": "2026-11-10T19:28:45.287Z", "ms": 630384, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "c8914e7c-d4ba-4e67-aff4-bee0a23fd04c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "evaluate", "activatedAt": "2026-11-10T19:28:45.289Z", "completedAt": "2026-11-10T19:45:23.310Z", "ms": 998021, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "c8914e7c-d4ba-4e67-aff4-bee0a23fd04c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "finish", "activatedAt": "2026-11-10T19:45:23.311Z", "completedAt": "2026-11-10T19:51:02.150Z", "ms": 338839, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "b46702bd-d9bb-41e3-9e50-8b4dd6e74c8b" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T18:52:20.223Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T18:52:20.097Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 630384 }, "sessions": { "total": 6, "taskScoped": 4, "byGezel": { "efua": 1, "kenji": 1, "pimchan": 1, "santosh": 2, "sibusiso": 1 }, "perTask": { "fieldstone-launch-comms/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 10, "maxContextFill": 0.2806396484375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2806396484375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 1, "tasks": 3, "tasksWithReferences": 1, "items": 5, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/gezel-in-docker", "knowledge://bendyline/handboek/notifications", "knowledge://bendyline/handboek/adding-folders", "knowledge://bendyline/handboek/craftbook/chapter-weave" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "invoked the bundled press-release recipe, ran the task to completion, and the release carries a dateline, the attributed approved quote, the boilerplate, a contact, and the approved facts rather than the stale draft", "durationMs": 3533372, "timeoutMs": 7261725, "budgetUsedFraction": 0.4865747463584754 }, "timing": { "startedAt": "2026-11-10T18:52:14.367Z", "finishedAt": "2026-11-10T19:51:07.739Z", "timeToFirstArtifactMs": 507896, "timeToLastArtifactWriteMs": 3188773, "timeToFirstTokenMs": 56567, "firstTurnTtftMs": 37239, "timeToFirstToolCallMs": 98523 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Planner", "Copywriter", "Reviewer", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 47, "byTool": { "suggest_work": 1, "start_work": 1, "read_task_notes": 5, "get_task": 5, "search": 1, "message_gezel": 1, "read_file": 16, "list_dir": 1, "write_artifact": 3, "write_file": 4, "advance_task_step": 4, "replace_in_file": 1, "stat": 2, "read_artifact": 1, "write_task_note": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 7 }, "sniff": { "progression": [ { "atMs": 7357, "filePath": "craftbook-route-multi", "bytes": 0, "score": 9, "scoreMax": 11, "signals": [], "failReason": "no task has been started from the bundled \"press-release\" recipe yet — search the recipe library for the one that covers writing an announcement for journalists and invoke it on this project" }, { "atMs": 126774, "filePath": "craftbook-route-multi", "bytes": 0, "score": 9, "scoreMax": 11, "signals": [], "failReason": "task fieldstone-launch-comms/1 (from \"press-release\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 831650, "filePath": "craftbook-route-multi", "bytes": 1147, "score": 10, "scoreMax": 11, "signals": [], "failReason": "task fieldstone-launch-comms/1 (from \"press-release\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 3528181, "filePath": "craftbook-route-multi", "bytes": 1147, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-route-multi", "bytes": 1147, "score": 11, "scoreMax": 11, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 3528179, "kind": "structured", "gezel": "kenji", "question": "Your document is finished. - **`press-release.md`** - `tasks/1/verdict.md` - `o…", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-10T18:52:14.370Z] [trial] id=craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr scenario=craftbook-route-multi model=underdog-saluki-27b-iq2-mix", "[2026-11-10T18:52:14.370Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T18:52:14.370Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-route-multi-05YPC0", "[2026-11-10T18:52:14.370Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "[2026-11-10T18:52:14.370Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T18:52:14.372Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T18:52:14.376Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T18:52:14.381Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-route-multi-05YPC0", "[2026-11-10T18:52:14.382Z] [trial] throughput-scaled ceiling: 80m → 121m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T18:52:20.194Z] [trial] daemon spawned pid=39804 port=https://127.0.0.1:64727", "[2026-11-10T18:52:21.165Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T18:52:21.184Z] [trial] running scenario setup hook", "[2026-11-10T18:52:21.183Z] [trial] meester=kenji", "[2026-11-10T18:52:21.476Z] [trial] scenario setup complete", "[2026-11-10T18:52:21.478Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T18:52:21.551Z] [poll] initial digests hard=6c38c8bff37fd5ce soft=50ba3bc3fd676594", "[2026-11-10T18:52:21.814Z] [poll] hard-progress: 6c38c8bff37fd5ce → 8c970777fb11e983 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=9/0B)", "[2026-11-10T18:53:57.428Z] [poll] hard-progress: 8c970777fb11e983 → fe2a10bcfe21dc61 after 96s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=134,stream=415) sniff=9/0B)", "[2026-11-10T18:54:21.209Z] [poll] hard-progress: fe2a10bcfe21dc61 → 0ae1752ffdda497b after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=186,stream=672) sniff=9/0B)", "[2026-11-10T18:54:33.071Z] [poll] hard-progress: 0ae1752ffdda497b → 742fe608e4bd69f2 after 12s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=211,stream=749) sniff=9/0B)", "[2026-11-10T18:54:38.826Z] [poll] hard-progress: 742fe608e4bd69f2 → 7602169576ec7b9a after 6s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=4,slot=219,stream=776) sniff=9/0B)", "[2026-11-10T18:55:26.398Z] [poll] hard-progress: 7602169576ec7b9a → 58c6937a77f4a57e after 48s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=290,stream=976) sniff=9/0B)", "[2026-11-10T18:55:38.327Z] [poll] hard-progress: 58c6937a77f4a57e → f6c76afae5bc8daa after 12s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=5,slot=310,stream=1035) sniff=9/0B)", "[2026-11-10T18:56:20.416Z] [poll] hard-progress: f6c76afae5bc8daa → deb9ca74bd0f8871 after 42s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=7,slot=363,stream=1182) sniff=9/0B)", "[2026-11-10T18:56:56.041Z] [poll] hard-progress: deb9ca74bd0f8871 → a026c20280fb5a0d after 36s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=442,stream=1417) sniff=9/0B)" ] }, "metrics": { "process": { "sampleCount": 589, "peakRssMb": 14143, "peakCpuPercent": 1372.2, "firstAtMs": 1, "lastAtMs": 3521164 }, "gpu": { "available": true, "sampleCount": 589, "peakUtilPercent": 100, "peakMemUsedMb": 10642, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 589, "peakUsedMb": 30766, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 151149, "totalOutputTokens": 24430, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 9, "todayTokensIn": 148048, "todayTokensCached": 144983, "todayTokensOut": 4072, "todayCost": 0, "totalTurns": 9, "totalTokensIn": 148048, "totalTokensCached": 144983, "totalTokensOut": 4072, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 9 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 9666, "cachedInputTokens": 9282, "outputTokens": 177, "cost": 0, "durationMs": 335941, "outputTokensPerSec": 8.406529848482808, "at": "2026-11-10T19:51:02.232Z" }, "lastUpdated": "2026-11-10T19:51:02.232Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T19:51:02.742Z" } }, "source": "engine-log" }, "derived": { "durationMs": 3522535, "meanTokensPerSec": 8.4, "genTokensPerSec": 8.4, "genTokens": 24430, "promptTokensPerSec": 334.88, "peakRssMb": 14143, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 148048, "output": 4072 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T18:52:14.367Z", "finishedAt": "2026-11-10T19:51:07.739Z", "timeToFirstArtifactMs": 507896, "timeToLastArtifactWriteMs": 3188773, "timeToFirstTokenMs": 56567, "firstTurnTtftMs": 37239, "timeToFirstToolCallMs": 98523 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 3 }, "steps": { "activated": 7, "activationEvents": 5, "completed": 5, "gateApprovals": 5, "gateRejections": 1, "redrives": 0, "perStep": [ { "ref": "fieldstone-launch-comms/1", "stepId": "scope", "activatedAt": "2026-11-10T18:54:15.866Z", "completedAt": "2026-11-10T19:00:42.459Z", "ms": 386593, "gateRejections": 0, "toolCalls": 3, "sessionIds": [ "df3d4cb8-4d73-47c2-9b55-5bafdd4762f2" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "draft", "activatedAt": "2026-11-10T19:07:10.326Z", "completedAt": "2026-11-10T19:18:14.902Z", "ms": 664576, "gateRejections": 1, "toolCalls": 6, "sessionIds": [ "945af339-4090-4423-a2f2-a918f860fa8c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "review", "activatedAt": "2026-11-10T19:18:14.903Z", "completedAt": "2026-11-10T19:28:45.287Z", "ms": 630384, "gateRejections": 0, "toolCalls": 5, "sessionIds": [ "c8914e7c-d4ba-4e67-aff4-bee0a23fd04c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "evaluate", "activatedAt": "2026-11-10T19:28:45.289Z", "completedAt": "2026-11-10T19:45:23.310Z", "ms": 998021, "gateRejections": 0, "toolCalls": 6, "sessionIds": [ "c8914e7c-d4ba-4e67-aff4-bee0a23fd04c" ] }, { "ref": "fieldstone-launch-comms/1", "stepId": "finish", "activatedAt": "2026-11-10T19:45:23.311Z", "completedAt": "2026-11-10T19:51:02.150Z", "ms": 338839, "gateRejections": 0, "toolCalls": 9, "sessionIds": [ "b46702bd-d9bb-41e3-9e50-8b4dd6e74c8b" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T18:52:20.223Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T18:52:20.097Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 630384 }, "sessions": { "total": 6, "taskScoped": 4, "byGezel": { "efua": 1, "kenji": 1, "pimchan": 1, "santosh": 2, "sibusiso": 1 }, "perTask": { "fieldstone-launch-comms/1": 4 }, "reusedAcrossSteps": 1, "continuityReuses": 1, "continuityBreaks": 0, "sessionsPerStep": 0.5714285714285714, "maxMessages": 10, "maxContextFill": 0.2806396484375, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2806396484375 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-complex-work-craftbook-route-multi", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-route-multi", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "scenarioId": "craftbook-route-multi", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-10T20:31:04.431Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/out/press-release.md", "artifactBytes": 1149, "artifactSha256": "f532e8eff0c9e6c7e5b47c68d3e4870c6bd7b7a12ce3493be7ec242e724f4569", "axisSource": "evals/src/craftbooks/authoring/route-multi.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "The bundled press-release workflow finishes and the output preserves approved facts, quote wording, boilerplate and contact while ignoring the stale teaser. Minor defects remain: quote punctuation changes and contact follows the requested final boilerplate. Root/out copies differ, so review and delivered artifact are not identical.", "reviewBasis": "Read declared artifacts with20KiB cap, relevant seeded sources, tool receipts and frozen facts. Additional runtime checks when cited are advisory; no model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/out/press-release.md", "status": "notApplicable", "bytes": 1149, "sha256": "f532e8eff0c9e6c7e5b47c68d3e4870c6bd7b7a12ce3493be7ec242e724f4569", "reviewedBytes": 1149, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/press-release.md", "status": "notApplicable", "bytes": 1091, "sha256": "f697ad2803394b9c27ab4973c20b503ba629389da497651a08c7b002958873b2", "reviewedBytes": 1091, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/facts/announcement.md", "status": "notApplicable", "bytes": 1005, "sha256": "6e19552f74df994beccd4b45ff25a0a8feb92aef7ce5e918782f38ef22d6a39f", "reviewedBytes": 1005, "truncated": false, "role": "approvedSource" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/artifacts/fieldstone-launch-comms/tasks/1/scope.md", "status": "notApplicable", "bytes": 2626, "sha256": "59984e1d0d3a23fc89ae2f2dcfb668be396cff9f288c4750a0293ee728029f15", "reviewedBytes": 2626, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/artifacts/fieldstone-launch-comms/tasks/1/verdict.md", "status": "notApplicable", "bytes": 1412, "sha256": "4d87cdaacbc894d2f57108e91ca73ed23309471f8435a613eb0231f9b4a41e5e", "reviewedBytes": 1412, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T18:52:22.876Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T18:52:34.929Z INFO [chat] [llama-server] launch {\"pid\":47896,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T19:51:06.948Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-47896-1794340266947\",\"pid\":47896,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":3512018,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 39669, "peakGpuMiB": 10642, "peakProcessRssMiB": 14143, "peakSystemRamMiB": 30766, "decodeTokensPerSec": 8.4, "prefillTokensPerSec": 334.88, "generatedTokens": 24430, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.2806396484375 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:07:25.064Z", "reviewId": "125", "cellId": "r3-saluki-cuda-complex-work-craftbook-route-multi", "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "scenarioId": "craftbook-route-multi", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-route-multi", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "c307dae1f2e471121712f7f5c4fb7f0b308d2c34d6a9fa597dd83ece93afce39", "evidenceSha256": "d82d1d4fd40c8121513ee250aa1253eecba7b5aa657239feedc78f99794dc2f0", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/out/press-release.md", "bytes": 1149, "sha256": "f532e8eff0c9e6c7e5b47c68d3e4870c6bd7b7a12ce3493be7ec242e724f4569", "shownBytes": 1149, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/press-release.md", "bytes": 1091, "sha256": "f697ad2803394b9c27ab4973c20b503ba629389da497651a08c7b002958873b2", "shownBytes": 1091, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/facts/announcement.md", "bytes": 1005, "sha256": "6e19552f74df994beccd4b45ff25a0a8feb92aef7ce5e918782f38ef22d6a39f", "shownBytes": 1005, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/artifacts/fieldstone-launch-comms/tasks/1/scope.md", "bytes": 2626, "sha256": "59984e1d0d3a23fc89ae2f2dcfb668be396cff9f288c4750a0293ee728029f15", "shownBytes": 2626, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/artifacts/fieldstone-launch-comms/tasks/1/verdict.md", "bytes": 1412, "sha256": "4d87cdaacbc894d2f57108e91ca73ed23309471f8435a613eb0231f9b4a41e5e", "shownBytes": 1412, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No axes declared. Qualitatively, out/press-release.md matches the approved facts: verbatim quote and boilerplate, correct contact, managing-director title, dateline, ### mark, no teaser content. The task's scope.md and verdict.md show it was driven through a recipe. The release is a clean, factual draft.", "limitations": "Tool calls (suggest_craftbook, invoke, no start_project) can't be verified from the evidence. The dateline date \"Nov. 10, 2026\" is not in the approved facts. A stray press-release.md at the project root has \"Oct. 14\" and differs from the output. The verdict cites that root file.", "meanScore": null, "durationMs": 15313, "costUsd": 0.023932, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 5147, "cache_read_input_tokens": 700, "output_tokens": 327, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 5147, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 327, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 5147, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5147 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 327, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 5147, "webSearchRequests": 0, "costUSD": 0.023932, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 5147, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-route-multi / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 58m 53s; budget used: 48.7%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 507896`; `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":1147,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.4865747463584754`; `facts.toolUse.totalToolCalls: 47`; `facts.timing.timeToFirstArtifactMs: 507896`; `facts.outcome.durationMs: 3533372` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14143 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10642 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30766 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.4 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 151149 + 24430 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 3s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 7 → 5 | `continuity.steps` |\n| Gate rejections / redrives | 1 / 0 | `continuity.steps` |\n| Median step wall-clock | 630s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 4 / 0.57 | `continuity.sessions` |\n| Sessions reused across steps | 1 (1 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 1 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 28% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 1 / 1 of 3 / 5 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 507896`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":1147,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.4865747463584754`\n- `facts.toolUse.totalToolCalls: 47`\n- `facts.timing.timeToFirstArtifactMs: 507896`\n- `facts.outcome.durationMs: 3533372`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 507896`; `facts.sniff.latest: {\"filePath\":\"craftbook-route-multi\",\"bytes\":1147,\"score\":11,\"scoreMax\":11,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.4865747463584754`; `facts.toolUse.totalToolCalls: 47`; `facts.timing.timeToFirstArtifactMs: 507896`; `facts.outcome.durationMs: 3533372`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.5/10**, pass; unchanged. Advisory **notApplicable**. The bundled press-release workflow finishes and the output preserves approved facts, quote wording, boilerplate and contact while ignoring the stale teaser. Minor defects remain: quote punctuation changes and contact follows the requested final boilerplate. Root/out copies differ, so review and delivered artifact are not identical.\n\n\n\n## Positives\n\nThe model selects an existing press-release recipe and actually completes its multi-role workflow. The approved October14 opening, Rotterdam location,2019 founding,11employees, two open evenings monthly and68%local-order figure are preserved; managing director replaces the stale CEO claim. No new recipe is unnecessarily authored.\n\n## Negatives and diagnosis\n\nTemplate gates expect root press-release.md while the scenario request expects out/press-release.md. After a path rejection both copies are written but differ: the reviewed root version and final output are not byte-identical. The quote’s final period becomes a comma before attribution, and a contact block follows boilerplate described as the final paragraph. A summary claim of verbatim quotation is therefore too strong. The subheading can ambiguously imply the location opens only twice monthly; body text clarifies open evenings.\n\n## Performance and fit\n\nDecode 8.4tok/s, prefill 334.88tok/s; 24430generated tokens. Peak GPU 10642/12227MiB, process RSS 14143MiB, whole-system RAM 30766/31865MiB. Context 65536, 1slot, f16; max observed released slot 39669. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nBind one explicit output-path parameter throughout the existing press-release template, including the reviewer and completion gate. Preserve approved quote punctuation and literal final-paragraph requirements, and identify the hash of the version reviewed.\n\n## Strategic experiment — not applied\n\nCarry the requested destination through suggest_work/start_work into all task outputs and gates. Link review receipts to artifact versions so later edits cannot silently inherit approval. Test with unrelated reports and code-change notes.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/facts.json): pass, fixed 8.5, 34.51minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/facts.json): pass, fixed 8.5, 32.31minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/facts.json): fail, fixed 5.5, 5.05minutes; other historical baseline; contextual only.\n\nUnlike the narrow developer/find-versus-create routing probes, this finishes real production and review. Linear authoring produces exact reconciled data, whereas this narrative success still has provenance/version and formatting limits. Original and continuation baselines remain distinct.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nTest canonical destination and version-bound review across release writing, invoice summaries and developer verification in a separate baseline; expect fewer path repairs without weakening checks.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-complex-work-craftbook-export-generalize", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-export-generalize", "suite": "complex-work", "scenario": "craftbook-export-generalize", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T19:54:06.809Z", "finishedAt": "2026-11-10T22:25:01.172Z", "durationMs": 9054363, "success": true, "reason": "did the Q1 pass, generalized it into a saved reusable craftbook, ran that craftbook to completion, and the Q2 outputs are genuinely derived from the Q2 list", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2346288 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-export-generalize", "bytes": 3644, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 0, "ruleId": "E_BOTH_SEVERE", "summary": "The trial exhausted its budget while making at least thirty tool calls.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 1 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 114 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 2346288 }, { "source": "facts", "path": "outcome.durationMs", "value": 9054363 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8, "band": "ship-ready" }, "facts": { "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "scenarioId": "craftbook-export-generalize", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 1502, "peakRssMb": 12640, "peakCpuPercent": 1737.6, "firstAtMs": 0, "lastAtMs": 9041024 }, "gpu": { "available": true, "sampleCount": 1502, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1502, "peakUsedMb": 30903, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 494619, "totalOutputTokens": 61997, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 117128, "todayTokensCached": 115644, "todayTokensOut": 1169, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 117128, "totalTokensCached": 115644, "totalTokensOut": 1169, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11486, "cachedInputTokens": 11329, "outputTokens": 132, "cost": 0, "durationMs": 2621689, "outputTokensPerSec": 8.520004906482216, "at": "2026-11-10T22:24:53.380Z" }, "lastUpdated": "2026-11-10T22:24:53.380Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T22:24:54.817Z" } }, "source": "engine-log" }, "derived": { "durationMs": 9042589, "meanTokensPerSec": 8.42, "genTokensPerSec": 8.42, "genTokens": 61997, "promptTokensPerSec": 354.44, "peakRssMb": 12640, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 117128, "output": 1169 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 4 }, "steps": { "activated": 5, "activationEvents": 1, "completed": 2, "gateApprovals": 2, "gateRejections": 3, "redrives": 0, "perStep": [ { "ref": "supplier-intake/1", "stepId": "check", "activatedAt": "2026-11-10T20:32:00.247Z", "completedAt": null, "ms": null, "gateRejections": 3, "toolCalls": 6, "sessionIds": [ "9d1aa0aa-b0e7-4416-b43c-234c11bca658" ] }, { "ref": "supplier-intake/2", "stepId": "check", "activatedAt": "2026-11-10T21:32:42.501Z", "completedAt": "2026-11-10T21:40:05.958Z", "ms": 443457, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "3af8a95e-0cb7-493c-a4cf-6a47a5c108e9" ] }, { "ref": "supplier-intake/2", "stepId": "write", "activatedAt": "2026-11-10T21:40:05.961Z", "completedAt": "2026-11-10T22:24:53.285Z", "ms": 2687324, "gateRejections": 0, "toolCalls": 13, "sessionIds": [ "2105d22f-fedd-45ad-a62c-ec1c3293abad" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T19:54:12.417Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T19:54:12.335Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1565390.5 }, "sessions": { "total": 6, "taskScoped": 3, "byGezel": { "ambrose": 1, "dawit": 1, "leandro": 2, "martin": 1, "reza": 1 }, "perTask": { "supplier-intake/2": 2, "supplier-intake/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6, "maxMessages": 7, "maxContextFill": 0.432861328125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.432861328125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 2, "tasks": 4, "tasksWithReferences": 2, "items": 10, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/reorder-run", "knowledge://bendyline/handboek/craftbook/recipe-capture", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbook/receipt-intake", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/reorder-run", "knowledge://bendyline/handboek/craftbook/recipe-capture", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbook/receipt-intake" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "did the Q1 pass, generalized it into a saved reusable craftbook, ran that craftbook to completion, and the Q2 outputs are genuinely derived from the Q2 list", "durationMs": 9054363, "timeoutMs": 7261725, "budgetUsedFraction": 1 }, "timing": { "startedAt": "2026-11-10T19:54:06.809Z", "finishedAt": "2026-11-10T22:25:01.172Z", "timeToFirstArtifactMs": 2346288, "timeToLastArtifactWriteMs": 9046572, "timeToFirstTokenMs": 46829, "firstTurnTtftMs": 36712, "timeToFirstToolCallMs": 75643 }, "team": { "totalGezelsCreated": 8, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Builder", "Data-Analyst", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 114, "byTool": { "read_file": 21, "list_craftbooks": 1, "craftbook_read": 4, "craftbook_write": 3, "write_artifact": 5, "invoke_craftbook": 3, "list_tasks": 4, "read_task_notes": 5, "get_task": 6, "stat": 8, "advance_task_step": 6, "activate_task": 1, "set_task_status": 1, "create_task": 1, "search": 2, "read_artifact": 4, "list_project_gezels": 1, "ensure_gezel": 1, "search_memory": 1, "message_gezel": 1, "list_artifacts": 3, "manage_task": 2, "start_work": 4, "suggest_work": 1, "write_task_note": 3, "run_installed_script": 1, "write_file": 8, "run_nodejs_script": 8, "replace_in_file": 3, "list_dir": 2 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 18 }, "sniff": { "progression": [ { "atMs": 7022, "filePath": "craftbook-export-generalize", "bytes": 0, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 1797416, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 2277765, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 2804765, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 2810765, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 2871193, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 2962910, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 3182782, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 4388108, "filePath": "craftbook-export-generalize", "bytes": 17, "score": 1, "scoreMax": 6, "signals": [], "failReason": "out/q1-intake.md does not exist yet — write the short Q1 intake report there: what you checked and which suppliers have an impossible lead time" }, { "atMs": 5763830, "filePath": "craftbook-export-generalize", "bytes": 1630, "score": 3, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/1 (from craftbook \"supplier-intake\") has status \"canceled\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 5793729, "filePath": "craftbook-export-generalize", "bytes": 3243, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/1 (from craftbook \"supplier-intake\") has status \"canceled\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 5919030, "filePath": "craftbook-export-generalize", "bytes": 3243, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/2 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 6780912, "filePath": "craftbook-export-generalize", "bytes": 3645, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/2 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 6798854, "filePath": "craftbook-export-generalize", "bytes": 3644, "score": 5, "scoreMax": 6, "signals": [], "failReason": "task supplier-intake/2 (from craftbook \"supplier-intake\") has status \"active\" — advance THIS task to a terminal step with advance_task_step/set_task_status. Do NOT invoke the recipe again or create a second task: a new task starts the work over and does not count." }, { "atMs": 9047542, "filePath": "craftbook-export-generalize", "bytes": 3644, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-export-generalize", "bytes": 3644, "score": 6, "scoreMax": 6, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 9047512, "kind": "structured", "gezel": "reza", "question": "**Q2 supplier intake (fresh run on fixed recipe)** is finished. - **`supplier-i…", "chose": "Dismiss" } ] }, "miscEvents": [ "[2026-11-10T19:54:06.812Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "[2026-11-10T19:54:06.812Z] [trial] id=craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi scenario=craftbook-export-generalize model=underdog-saluki-27b-iq2-mix", "[2026-11-10T19:54:06.812Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-export-generalize-f4FSwm", "[2026-11-10T19:54:06.812Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T19:54:06.814Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T19:54:06.812Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T19:54:06.818Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T19:54:06.823Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-export-generalize-f4FSwm", "[2026-11-10T19:54:06.823Z] [trial] throughput-scaled ceiling: 80m → 121m at 13.22 tok/s (reference 20 tok/s)", "[2026-11-10T19:54:12.310Z] [trial] daemon spawned pid=43552 port=https://127.0.0.1:53294", "[2026-11-10T19:54:13.242Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T19:54:13.259Z] [trial] meester=ambrose", "[2026-11-10T19:54:13.259Z] [trial] running scenario setup hook", "[2026-11-10T19:54:13.584Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T19:54:13.582Z] [trial] scenario setup complete", "[2026-11-10T19:54:13.654Z] [poll] initial digests hard=c2c522a923aba959 soft=ea7b1224e990bfb7", "[2026-11-10T19:54:13.946Z] [poll] hard-progress: c2c522a923aba959 → ebaa25927a16cf45 after 0s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=1/0B)", "[2026-11-10T19:55:24.767Z] [poll] hard-progress: ebaa25927a16cf45 → 69e50fffde13d985 after 71s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=2,slot=85,stream=209) sniff=1/0B)", "[2026-11-10T20:03:47.904Z] [poll] hard-progress: 69e50fffde13d985 → a788129d25968430 after 503s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=4,slot=1459,stream=4263) sniff=1/0B)", "[2026-11-10T20:12:00.171Z] [poll] hard-progress: a788129d25968430 → 262694fe37d46e7d after 492s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=6,slot=2726,stream=8018) sniff=1/0B)", "[2026-11-10T20:24:04.298Z] [poll] hard-progress: 262694fe37d46e7d → abb6be28ab617c6f after 724s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=8,slot=4626,stream=15015) sniff=1/17B)", "[2026-11-10T20:24:34.608Z] [poll] hard-progress: abb6be28ab617c6f → e403c537fbf04118 after 30s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=4680,stream=15184) sniff=1/17B)", "[2026-11-10T20:24:53.158Z] [poll] hard-progress: e403c537fbf04118 → a9de88b2955e4b77 after 19s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=11,slot=4722,stream=15314) sniff=1/17B)", "[2026-11-10T20:25:24.391Z] [poll] hard-progress: a9de88b2955e4b77 → c542de07c2fb344c after 31s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=12,slot=4795,stream=15522) sniff=1/17B)", "[2026-11-10T20:26:07.277Z] [poll] retry-loop nudge deferred: reza/4f914b3a in supplier-intake for 1914s, last tool call 126s ago (+1 more) is already working; refusing to queue a potentially stale edit" ] }, "metrics": { "process": { "sampleCount": 1502, "peakRssMb": 12640, "peakCpuPercent": 1737.6, "firstAtMs": 0, "lastAtMs": 9041024 }, "gpu": { "available": true, "sampleCount": 1502, "peakUtilPercent": 100, "peakMemUsedMb": 10648, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 1502, "peakUsedMb": 30903, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 494619, "totalOutputTokens": 61997, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 6, "todayTokensIn": 117128, "todayTokensCached": 115644, "todayTokensOut": 1169, "todayCost": 0, "totalTurns": 6, "totalTokensIn": 117128, "totalTokensCached": 115644, "totalTokensOut": 1169, "totalCost": 0, "medianOutputTokensPerSec": 8.5, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.5, "turns": 6 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 11486, "cachedInputTokens": 11329, "outputTokens": 132, "cost": 0, "durationMs": 2621689, "outputTokensPerSec": 8.520004906482216, "at": "2026-11-10T22:24:53.380Z" }, "lastUpdated": "2026-11-10T22:24:53.380Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-10T22:24:54.817Z" } }, "source": "engine-log" }, "derived": { "durationMs": 9042589, "meanTokensPerSec": 8.42, "genTokensPerSec": 8.42, "genTokens": 61997, "promptTokensPerSec": 354.44, "peakRssMb": 12640, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 117128, "output": 1169 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T19:54:06.809Z", "finishedAt": "2026-11-10T22:25:01.172Z", "timeToFirstArtifactMs": 2346288, "timeToLastArtifactWriteMs": 9046572, "timeToFirstTokenMs": 46829, "firstTurnTtftMs": 36712, "timeToFirstToolCallMs": 75643 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 4 }, "steps": { "activated": 5, "activationEvents": 1, "completed": 2, "gateApprovals": 2, "gateRejections": 3, "redrives": 0, "perStep": [ { "ref": "supplier-intake/1", "stepId": "check", "activatedAt": "2026-11-10T20:32:00.247Z", "completedAt": null, "ms": null, "gateRejections": 3, "toolCalls": 6, "sessionIds": [ "9d1aa0aa-b0e7-4416-b43c-234c11bca658" ] }, { "ref": "supplier-intake/2", "stepId": "check", "activatedAt": "2026-11-10T21:32:42.501Z", "completedAt": "2026-11-10T21:40:05.958Z", "ms": 443457, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "3af8a95e-0cb7-493c-a4cf-6a47a5c108e9" ] }, { "ref": "supplier-intake/2", "stepId": "write", "activatedAt": "2026-11-10T21:40:05.961Z", "completedAt": "2026-11-10T22:24:53.285Z", "ms": 2687324, "gateRejections": 0, "toolCalls": 13, "sessionIds": [ "2105d22f-fedd-45ad-a62c-ec1c3293abad" ] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T19:54:12.417Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T19:54:12.335Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1565390.5 }, "sessions": { "total": 6, "taskScoped": 3, "byGezel": { "ambrose": 1, "dawit": 1, "leandro": 2, "martin": 1, "reza": 1 }, "perTask": { "supplier-intake/2": 2, "supplier-intake/1": 1 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6, "maxMessages": 7, "maxContextFill": 0.432861328125, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.432861328125 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-complex-work-craftbook-export-generalize", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-export-generalize", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "scenarioId": "craftbook-export-generalize", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-10T23:03:43.079Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q2-intake.md", "artifactBytes": 762, "artifactSha256": "95287311053075cc2d18b87aed6e8682b26ede21e1310d0cff30608e8bd39467", "axisSource": "evals/src/craftbooks/authoring/export-generalize.ts; no declared judge axes", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "The saved recipe finishes and Q2 reporting correctly flags both impossible lead times. However Q1 misses the zero-day supplier, Q2 clean.json exactly copies its raw input with0/-14 still present, and cleaning policies conflict. The fixed pass proves workflow completion and distinct quarter outputs, not reliable data cleaning.", "reviewBasis": "Read all four declared outputs, both source lists, saved-recipe tool receipts, support scripts and working artifacts with20KiB per-file cap. Independently compared parsed records and checked script syntax without executing generated scripts. No model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q2-intake.md", "status": "notApplicable", "bytes": 762, "sha256": "95287311053075cc2d18b87aed6e8682b26ede21e1310d0cff30608e8bd39467", "reviewedBytes": 762, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q2-clean.json", "status": "notApplicable", "bytes": 1263, "sha256": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "reviewedBytes": 1263, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q1-intake.md", "status": "notApplicable", "bytes": 514, "sha256": "31380a2646fb01b6026d51f4ce10e5e9a37fc8b09a11a163714654b454acf9cb", "reviewedBytes": 514, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q1-clean.json", "status": "notApplicable", "bytes": 1102, "sha256": "5afd1ac820f6351024084c81a0821378c962dbc520a5a0c90b987645f229138a", "reviewedBytes": 1102, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/data/suppliers-q1.json", "status": "notApplicable", "bytes": 1260, "sha256": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "reviewedBytes": 1260, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/data/suppliers-q2.json", "status": "notApplicable", "bytes": 1263, "sha256": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "reviewedBytes": 1263, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/craftbooks/supplier-intake.json", "status": "notApplicable", "bytes": 1725, "sha256": "b5537301eda4b706a5c718155fb96334794fb40d407ed9e6e112d835a44ccc45", "reviewedBytes": 1725, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/intake.js", "status": "notApplicable", "bytes": 2495, "sha256": "3278416fa9bf743d440f925749a5b3d12ce90ddcb97e09f86ba4f5b5c349a64e", "reviewedBytes": 2495, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/intake.cjs", "status": "notApplicable", "bytes": 2797, "sha256": "9323ab99791cbb17f70331e97ee8fe4cf73ef078dd96a0633771183878b3abf4", "reviewedBytes": 2797, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/verify-q2-clean.js", "status": "notApplicable", "bytes": 1346, "sha256": "1fa3259da0fb0887ef7cdf81870b0d6cabb4c4dedb476920585907b33f655980", "reviewedBytes": 1346, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/findings.md", "status": "notApplicable", "bytes": 867, "sha256": "f69ab37771c35fe8de965e7517a1892b738edc36513b30bc5abeb1d594b1a389", "reviewedBytes": 867, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/out/q1-intake.md", "status": "notApplicable", "bytes": 959, "sha256": "7002c8fc9fb57c7a18ae7a37dea0887c20a19780a7369364bc2e7d62f36a7223", "reviewedBytes": 959, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/out/q1-clean.json", "status": "notApplicable", "bytes": 1260, "sha256": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "reviewedBytes": 1260, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/supplier-intake/q2-findings.md", "status": "notApplicable", "bytes": 640, "sha256": "436f5184875b029cb6d53d94811a37f048acf1f8ae4fc8b918edd9884af95958", "reviewedBytes": 640, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T19:54:14.808Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T19:54:18.241Z INFO [chat] [llama-server] launch {\"pid\":34488,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-10T22:25:00.369Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-34488-1794349500369\",\"pid\":34488,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":9042128,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 42898, "peakGpuMiB": 10648, "peakProcessRssMiB": 12640, "peakSystemRamMiB": 30903, "decodeTokensPerSec": 8.42, "prefillTokensPerSec": 354.44, "generatedTokens": 61997, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.432861328125 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:07:30.984Z", "reviewId": "126", "cellId": "r3-saluki-cuda-complex-work-craftbook-export-generalize", "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "scenarioId": "craftbook-export-generalize", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-export-generalize", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "27dcfbfa61eae7586d0827c0dd6f16d44a0ec6876f830e8fff84dce1d9536774", "evidenceSha256": "82afa3708f56895071bc6b0476313c80ba11d85b08154fcacc48644685a270c3", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q2-intake.md", "bytes": 762, "sha256": "95287311053075cc2d18b87aed6e8682b26ede21e1310d0cff30608e8bd39467", "shownBytes": 762, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q2-clean.json", "bytes": 1263, "sha256": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "shownBytes": 1263, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q1-intake.md", "bytes": 514, "sha256": "31380a2646fb01b6026d51f4ce10e5e9a37fc8b09a11a163714654b454acf9cb", "shownBytes": 514, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q1-clean.json", "bytes": 1102, "sha256": "5afd1ac820f6351024084c81a0821378c962dbc520a5a0c90b987645f229138a", "shownBytes": 1102, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/data/suppliers-q1.json", "bytes": 1260, "sha256": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "shownBytes": 1260, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/data/suppliers-q2.json", "bytes": 1263, "sha256": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "shownBytes": 1263, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/craftbooks/supplier-intake.json", "bytes": 1725, "sha256": "b5537301eda4b706a5c718155fb96334794fb40d407ed9e6e112d835a44ccc45", "shownBytes": 1725, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/intake.js", "bytes": 2495, "sha256": "3278416fa9bf743d440f925749a5b3d12ce90ddcb97e09f86ba4f5b5c349a64e", "shownBytes": 2495, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/intake.cjs", "bytes": 2797, "sha256": "9323ab99791cbb17f70331e97ee8fe4cf73ef078dd96a0633771183878b3abf4", "shownBytes": 2797, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/verify-q2-clean.js", "bytes": 1346, "sha256": "1fa3259da0fb0887ef7cdf81870b0d6cabb4c4dedb476920585907b33f655980", "shownBytes": 1346, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/findings.md", "bytes": 867, "sha256": "f69ab37771c35fe8de965e7517a1892b738edc36513b30bc5abeb1d594b1a389", "shownBytes": 867, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/out/q1-intake.md", "bytes": 959, "sha256": "7002c8fc9fb57c7a18ae7a37dea0887c20a19780a7369364bc2e7d62f36a7223", "shownBytes": 959, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/out/q1-clean.json", "bytes": 1260, "sha256": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "shownBytes": 1260, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/supplier-intake/q2-findings.md", "bytes": 640, "sha256": "436f5184875b029cb6d53d94811a37f048acf1f8ae4fc8b918edd9884af95958", "shownBytes": 640, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "All four output files exist. Q1 report flags Alpine Cork (-5) but misses Norrland Textiles (0), and q1-clean.json drops only Alpine. Q2 outputs flag 0 and -14 and differ from Q1, but the craftbook threshold<0 rule disagrees with the Q2 report's >=1 rule. The craftbook has 3 steps. Q2 was likely not generated by the recipe.", "limitations": "The task-graph and craftbook catalog invocation evidence isn't shown. I can't confirm export_task_craftbook was used or that the recipe produced the Q2 files. The artifact set mixes inconsistent versions (intake.js vs intake.cjs, duplicate Q1 reports), and the execution was runner-terminal.", "meanScore": null, "durationMs": 15112, "costUsd": 0.055644, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 12960, "cache_read_input_tokens": 700, "output_tokens": 373, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 12960, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 373, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 12960, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 12960 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 373, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 12960, "webSearchRequests": 0, "costUSD": 0.055644, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 12960, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-export-generalize / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.0 / 10** (band: ship-ready)\n**Outcome: success** (duration: 150m 54s; budget used: 100.0%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 2346288`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3644,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 0 / 10 | The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 114`; `facts.timing.timeToFirstArtifactMs: 2346288`; `facts.outcome.durationMs: 9054363` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 0 + 0.15 * 10 = **8.0**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 12640 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10648 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30903 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.42 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 494619 + 61997 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 4s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 5 → 2 | `continuity.steps` |\n| Gate rejections / redrives | 3 / 0 | `continuity.steps` |\n| Median step wall-clock | 1565s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 3 / 0.60 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 1 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 43% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 2 / 2 of 4 / 10 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 2346288`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3644,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_BOTH_SEVERE\n\nThe trial exhausted its budget while making at least thirty tool calls.\n\n- `facts.outcome.budgetUsedFraction: 1`\n- `facts.toolUse.totalToolCalls: 114`\n- `facts.timing.timeToFirstArtifactMs: 2346288`\n- `facts.outcome.durationMs: 9054363`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 2346288`; `facts.sniff.latest: {\"filePath\":\"craftbook-export-generalize\",\"bytes\":3644,\"score\":6,\"scoreMax\":6,\"signals\":[],\"failReason\":null}`\n- Efficiency: The trial exhausted its budget while making at least thirty tool calls. `facts.outcome.budgetUsedFraction: 1`; `facts.toolUse.totalToolCalls: 114`; `facts.timing.timeToFirstArtifactMs: 2346288`; `facts.outcome.durationMs: 9054363`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8/10**, pass; unchanged. Advisory **notApplicable**. The saved recipe finishes and Q2 reporting correctly flags both impossible lead times. However Q1 misses the zero-day supplier, Q2 clean.json exactly copies its raw input with0/-14 still present, and cleaning policies conflict. The fixed pass proves workflow completion and distinct quarter outputs, not reliable data cleaning.\n\n\n\n## Positives\n\nThe authored two-step recipe accepts quarter/input/output parameters and is executed to a completed task. Q2 report and findings identify Trollhaugen Wool0 and Zagreb Enamel−14 accurately, with no Q1 suppliers copied into Q2. A real verifier execution passes its narrow count/field-order/trim checks. Final scripts parse under static syntax inspection; earlier ESM errors are repaired within the trial.\n\n## Negatives and diagnosis\n\nIndependent JSON comparison finds Q1 output is an unchanged seven-record source subset that removes−5 but keeps0; its workspace report misses that second anomaly. Q2 output is an exact eight-record copy of raw input, retaining both impossible values. The actual saved recipe explicitly keeps flagged suppliers and its verifier demands their presence, so it generalizes a retention/reporting policy rather than repairing invalid records. A stale workspace recipe copy instead says remove flagged records and uses a negative-only threshold. The scenario checks nonempty reports, parseable record lists, differing quarters and task completion, not these cleaning semantics. The first authored recipe references undefined workPath and pauses at its gate; repeated invocations reuse the obsolete task after edits, then the model cancels it and creates a fresh task. Several start_work calls cannot find the same recipe. Artifact/workspace confusion, two ESM failures, ineffective script-as-command/path calls and114tool calls add overhead. The agent rewrites intake.cjs after its last successful execution, so that final version is not runtime-verified. Preserve fixed8/pass; no operator intervention or retry occurred.\n\n## Performance and fit\n\nDecode 8.42tok/s, prefill 354.44tok/s; 61997generated tokens. Peak GPU 10648/12227MiB, process RSS 12640MiB, whole-system RAM 30903/31865MiB. Context 65536, 1slot, f16; max observed released slot 42898. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn packages/service/src/model-profile/behaviors/prompt-meester-craftbook-prelude.ts, experiment with one explicit validation/cleaning policy shared by the original pass, authored recipe and verifier. Require unresolved values to be retained as explicitly quarantined records or removed with an audit trail, never silently relabeled clean. Keep source and artifact paths explicit and use the supported ESM/script-file API.\n\n## Strategic experiment — not applied\n\nIn packages/service/src/http/routes/craftbooks.ts and tasks/craftbook-instantiation.ts, reject undefined template parameters before accepting a recipe; in packages/mcp/src/craftbook-invocation-idempotency.ts, evaluate how canceled or changed-recipe tasks should interact with deduplication. Separately strengthen future semantic cleanup evaluation to reconcile retained/dropped/repaired records; do not change this frozen gate or rescore the run.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/facts.json): fail, fixed 4.8, 400.84minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/facts.json): fail, fixed 2.4, 69.62minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/facts.json): fail, fixed 3.3, 13.63minutes; other historical baseline; contextual only.\n\nR3 author-linear independently reconciles all29rows, while this pass leaves invalid data intact despite successful workflow completion. Data-wrangle likewise demonstrates exact transformation; records-intake produces no output. These contrasting cases show why fixed success must be paired with artifact inspection. Earlier same-scenario results are contextual across the explicit continuation environment boundary.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nIn a fresh baseline, test one shared cleaning policy plus pre-save parameter validation across supplier intake, CSV cleanup and an unrelated report recipe; measure semantic defects, duplicate tasks and repair overhead without altering current evidence.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r3-saluki-cuda-complex-work-craftbook-author-params", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-params", "suite": "complex-work", "scenario": "craftbook-author-params", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "runner-terminal", "eligible": true, "success": true, "result": { "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "generalistMode": "auto", "engine": "llama-cpp", "startedAt": "2026-11-10T22:28:51.913Z", "finishedAt": "2026-11-11T00:03:05.110Z", "durationMs": 5653197, "success": true, "reason": "authored one parameterized craftbook, ran it twice with different region inputs, and both region summaries exist with distinct contents", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "engineContext": { "launches": 1, "grantedPerSlotTokens": 65536, "totalTokens": 65536, "slots": 1, "kvCacheType": "f16", "launchModel": "underdog-saluki-27b-iq2-mix" } }, "score": { "rubricVersion": "eval-run-fixed-v1", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "eligibility": { "failureClass": "pass", "includedInModelAggregate": true, "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "qualityProfile": {}, "axes": { "completion": { "score": 10, "ruleId": "C_SUCCESS", "summary": "The scenario reached a successful terminal verdict.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true } ] }, "quality": { "score": 10, "ruleId": "Q_GATE_COMPLETE", "summary": "The scenario gate is complete.", "evidence": [ { "source": "facts", "path": "outcome.success", "value": true }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1468768 }, { "source": "facts", "path": "sniff.latest", "value": { "filePath": "craftbook-author-params", "bytes": 323, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } } ] }, "efficiency": { "score": 2.5, "ruleId": "E_ONE_SEVERE", "summary": "Exactly one efficiency dimension reached the severe threshold.", "evidence": [ { "source": "facts", "path": "outcome.budgetUsedFraction", "value": 0.5019829544854162 }, { "source": "facts", "path": "toolUse.totalToolCalls", "value": 80 }, { "source": "facts", "path": "timing.timeToFirstArtifactMs", "value": 1468768 }, { "source": "facts", "path": "outcome.durationMs", "value": 5653197 } ] }, "behavior": { "score": 10, "ruleId": "B_CLEAN_LOW_INTERVENTION", "summary": "No red flags or missing roles were observed, with at most one intervention.", "evidence": [ { "source": "facts", "path": "toolUse.redFlags", "value": [] }, { "source": "facts", "path": "team.missingExpectedRoles", "value": [] }, { "source": "facts", "path": "autoAnswer.total", "value": 1 } ] } }, "composite": 8.5, "band": "ship-ready" }, "facts": { "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "scenarioId": "craftbook-author-params", "modelId": "underdog-saluki-27b-iq2-mix", "modelTier": "medium", "runDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "perf": { "process": { "sampleCount": 943, "peakRssMb": 14413, "peakCpuPercent": 1239, "firstAtMs": 0, "lastAtMs": 5640633 }, "gpu": { "available": true, "sampleCount": 943, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 943, "peakUsedMb": 30878, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 225830, "totalOutputTokens": 39447, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 172991, "totalTokensCached": 170325, "totalTokensOut": 7731, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 10 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8147, "cachedInputTokens": 7958, "outputTokens": 78, "cost": 0, "durationMs": 1074275, "outputTokensPerSec": 8.499300573791094, "at": "2026-11-10T23:56:42.135Z" }, "lastUpdated": "2026-11-10T23:56:42.135Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-11T00:02:59.641Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5642367, "meanTokensPerSec": 8.36, "genTokensPerSec": 8.36, "genTokens": 39447, "promptTokensPerSec": 327.68, "peakRssMb": 14413, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 172991, "output": 7731 } } } }, "continuity": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 6 }, "steps": { "activated": 9, "activationEvents": 5, "completed": 3, "gateApprovals": 3, "gateRejections": 3, "redrives": 0, "perStep": [ { "ref": "regional-sales-rollup/1", "stepId": "compute", "activatedAt": "2026-11-10T22:56:43.295Z", "completedAt": null, "ms": null, "gateRejections": 3, "toolCalls": 8, "sessionIds": [ "c131086a-934c-42f7-b8f2-2f318a44d247" ] }, { "ref": "regional-sales-rollup/2", "stepId": "compute", "activatedAt": "2026-11-10T22:46:26.921Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "281dcdfc-ff61-4c13-aa78-7f846f2e0a4c" ] }, { "ref": "regional-sales-rollup/3", "stepId": "compute", "activatedAt": "2026-11-10T23:21:11.963Z", "completedAt": "2026-11-10T23:29:16.016Z", "ms": 484053, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "2cb64035-e6e2-47bb-9b9b-ed6c453f0bfe" ] }, { "ref": "regional-sales-rollup/3", "stepId": "write-summary", "activatedAt": "2026-11-10T23:29:16.017Z", "completedAt": "2026-11-10T23:56:42.093Z", "ms": 1646076, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "bd2a070f-bdb8-4b05-88c3-9d7391b5e259" ] }, { "ref": "regional-sales-rollup/4", "stepId": "compute", "activatedAt": "2026-11-10T23:21:12.324Z", "completedAt": "2026-11-10T23:38:42.862Z", "ms": 1050538, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "c47d1748-91f2-4d56-9fd5-9fb836986a32" ] }, { "ref": "regional-sales-rollup/4", "stepId": "write-summary", "activatedAt": "2026-11-10T23:38:42.864Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "aba6342d-07ec-4adf-b2dd-23b7fd06c050" ] }, { "ref": "regional-sales-rollup/3", "stepId": "finish", "activatedAt": "2026-11-10T23:56:42.094Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T22:28:57.389Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T22:28:57.269Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1050538 }, "sessions": { "total": 9, "taskScoped": 6, "byGezel": { "margarida": 5, "mina": 3, "reza": 1 }, "perTask": { "regional-sales-rollup/2": 1, "regional-sales-rollup/3": 2, "regional-sales-rollup/1": 1, "regional-sales-rollup/4": 2 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6666666666666666, "maxMessages": 7, "maxContextFill": 0.5150299072265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5150299072265625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "retrieval": { "arm": null, "policy": { "modes": {}, "inheritedFrom": {} }, "turn": { "probes": 0, "injections": 0, "hitsBySource": {}, "injectedTokens": 0, "rejected": {}, "sessionsWithInjection": 0, "relevance": {} }, "references": { "searches": 4, "tasks": 6, "tasksWithReferences": 4, "items": 20, "citations": [ "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbook/cookbook-layout", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "shared:About this library.md", "knowledge://bendyline/handboek/craftbook/csv-transformer", "knowledge://bendyline/handboek/craftbook/cookbook-layout", "knowledge://bendyline/handboek/craftbook/automation-recipe", "knowledge://bendyline/handboek/craftbook/scrape-to-structured", "shared:About this library.md", "knowledge://bendyline/handboek/project-type/sales-pipeline", "knowledge://bendyline/handboek/craftbook/monthly-recording-review", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbook/month-close", "shared:About this library.md", "knowledge://bendyline/handboek/project-type/sales-pipeline", "knowledge://bendyline/handboek/craftbook/monthly-recording-review", "knowledge://bendyline/handboek/craftbook/invoice-run", "knowledge://bendyline/handboek/craftbook/month-close" ], "relevance": {} }, "stampedMessages": 0 }, "outcome": { "success": true, "reason": "authored one parameterized craftbook, ran it twice with different region inputs, and both region summaries exist with distinct contents", "durationMs": 5653197, "timeoutMs": 11261731, "budgetUsedFraction": 0.5019829544854162 }, "timing": { "startedAt": "2026-11-10T22:28:51.913Z", "finishedAt": "2026-11-11T00:03:05.110Z", "timeToFirstArtifactMs": 1468768, "timeToLastArtifactWriteMs": 5644369, "timeToFirstTokenMs": 47564, "firstTurnTtftMs": 38167, "timeToFirstToolCallMs": 54301 }, "team": { "totalGezelsCreated": 7, "rolesCreated": [ "Meester", "Scribe", "Librarian", "Crew Lead", "Data-Analyst", "Developer" ], "missingExpectedRoles": [] }, "toolUse": { "totalToolCalls": 80, "byTool": { "list_craftbooks": 2, "craftbook_read": 3, "read_file": 13, "craftbook_write": 4, "invoke_craftbook": 4, "list_tasks": 2, "read_task_notes": 10, "stat": 4, "get_task": 4, "list_artifacts": 3, "read_artifact": 6, "set_task_status": 2, "save_memory": 2, "message_gezel": 1, "list_dir": 5, "write_artifact": 10, "ask_user_question": 1, "write_file": 2, "write_task_note": 1, "advance_task_step": 1 }, "redFlags": [] }, "artifacts": { "htmlFiles": [], "imageFiles": [], "otherFileCount": 8 }, "sniff": { "progression": [ { "atMs": 6947, "filePath": "craftbook-author-params", "bytes": 0, "score": 4, "scoreMax": 7, "signals": [], "failReason": "no reusable authored craftbook with at least 2 steps exists yet — author the regional sales rollup as a reusable craftbook template, not as ad-hoc chat work" }, { "atMs": 1029562, "filePath": "craftbook-author-params", "bytes": 51, "score": 4, "scoreMax": 7, "signals": [], "failReason": "no task has been created from craftbook \"region-sales-rollup\" yet — start the recipe on this project for the north region" }, { "atMs": 1059500, "filePath": "craftbook-author-params", "bytes": 51, "score": 5, "scoreMax": 7, "signals": [], "failReason": "out/north-summary.md is 0 bytes, need ≥ 80" }, { "atMs": 4740670, "filePath": "craftbook-author-params", "bytes": 187, "score": 6, "scoreMax": 7, "signals": [], "failReason": "out/south-summary.md is 0 bytes, need ≥ 80" }, { "atMs": 5647250, "filePath": "craftbook-author-params", "bytes": 323, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } ], "latest": { "filePath": "craftbook-author-params", "bytes": 323, "score": 7, "scoreMax": 7, "signals": [], "failReason": null } }, "autoAnswer": { "total": 1, "byKind": { "structured": 1, "inline": 0 }, "events": [ { "atMs": 5257160, "kind": "structured", "gezel": "margarida", "question": "Both runs are blocked: writes to tasks/3/numbers.md and tasks/4/numbers.md are r…", "chose": "write-in (proceed-now)" } ] }, "miscEvents": [ "[2026-11-10T22:28:51.916Z] [trial] id=craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f scenario=craftbook-author-params model=underdog-saluki-27b-iq2-mix", "[2026-11-10T22:28:51.916Z] [trial] trialHome=\\AppData\\Local\\Temp\\gezel-eval-craftbook-author-params-OXCUr1", "[2026-11-10T22:28:51.916Z] [trial] cacheRoot=\\.gezel-eval-cache", "[2026-11-10T22:28:51.916Z] [trial] runDir=evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "[2026-11-10T22:28:51.916Z] [trial] provider=llama-cpp category=local-engine", "[2026-11-10T22:28:51.918Z] [trial] llama-server=packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe variant=env build=11960 rev=8642541b backend=cuda cuda-arch=60-virtual,70-real,75-real,80-real,86-real,89-real,90-real,100-real,120", "[2026-11-10T22:28:51.921Z] [trial] llama-cpp source=\\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix (read-only)", "[2026-11-10T22:28:51.925Z] [trial] cloned llama-cpp/underdog-saluki-27b-iq2-mix from \\.gezel-dev\\engines\\llama-cpp\\models\\underdog-saluki-27b-iq2-mix into \\AppData\\Local\\Temp\\gezel-eval-craftbook-author-params-OXCUr1", "[2026-11-10T22:28:51.925Z] [trial] throughput-scaled ceiling: 90m → 188m at 9.59 tok/s (reference 20 tok/s)", "[2026-11-10T22:28:57.324Z] [trial] daemon spawned pid=22012 port=https://127.0.0.1:53286", "[2026-11-10T22:28:58.257Z] [trial] provider=llama-cpp generalistMode=auto configured, firstRunCompleted=true", "[2026-11-10T22:28:58.276Z] [trial] running scenario setup hook", "[2026-11-10T22:28:58.276Z] [trial] meester=nadia", "[2026-11-10T22:28:58.621Z] [trial] scenario setup complete", "[2026-11-10T22:28:58.621Z] [trial] skipped initial meester prompt (scenario setup already kicked off work)", "[2026-11-10T22:28:58.695Z] [poll] initial digests hard=e6ae1842245c670f soft=8dc966bf6556e5b8", "[2026-11-10T22:28:59.011Z] [poll] hard-progress: e6ae1842245c670f → d54866cb8b1a73d6 after 0s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=0,tools=0,slot=0,stream=0) sniff=4/0B)", "[2026-11-10T22:29:52.102Z] [poll] hard-progress: d54866cb8b1a73d6 → 0eff1b4a0bcae213 after 53s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=1,slot=32,stream=47) sniff=4/0B)", "[2026-11-10T22:30:15.800Z] [poll] hard-progress: 0eff1b4a0bcae213 → bff6a89c2fd9f195 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=2,slot=58,stream=116) sniff=4/0B)", "[2026-11-10T22:32:32.610Z] [poll] hard-progress: bff6a89c2fd9f195 → e00a16734c49b0cb after 137s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=3,slot=423,stream=1173) sniff=4/0B)", "[2026-11-10T22:32:56.410Z] [poll] hard-progress: e00a16734c49b0cb → c5d675618240b0e0 after 24s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=5,slot=449,stream=1241) sniff=4/0B)", "[2026-11-10T22:41:55.527Z] [poll] hard-progress: c5d675618240b0e0 → 8344868a757ae7de after 539s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=6,slot=1886,stream=6824) sniff=4/0B)", "[2026-11-10T22:46:01.609Z] [poll] hard-progress: 8344868a757ae7de → 7cdc492021b8d7a0 after 246s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=7,slot=2514,stream=10027) sniff=4/51B)", "[2026-11-10T22:46:31.549Z] [poll] hard-progress: 7cdc492021b8d7a0 → 9cbd2d654d391da6 after 30s (workspaceProjects=3 sessions=1 daemon[src=service-telemetry](turns=1,tools=9,slot=2561,stream=10178) sniff=5/51B)", "[2026-11-10T22:47:37.492Z] [poll] hard-progress: 9cbd2d654d391da6 → 6a80e2e28f143ac1 after 66s (workspaceProjects=3 sessions=2 daemon[src=service-telemetry](turns=2,tools=9,slot=2651,stream=10424) sniff=5/51B)" ] }, "metrics": { "process": { "sampleCount": 943, "peakRssMb": 14413, "peakCpuPercent": 1239, "firstAtMs": 0, "lastAtMs": 5640633 }, "gpu": { "available": true, "sampleCount": 943, "peakUtilPercent": 100, "peakMemUsedMb": 10646, "memTotalMb": 12227, "memoryModel": "discrete" }, "systemMemory": { "sampleCount": 943, "peakUsedMb": 30878, "totalMb": 31865 }, "usage": { "available": true, "totalInputTokens": 225830, "totalOutputTokens": 39447, "totalCostUsd": 0, "rawProviders": { "llama-cpp": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 10, "totalTokensIn": 172991, "totalTokensCached": 170325, "totalTokensOut": 7731, "totalCost": 0, "medianOutputTokensPerSec": 8.4, "modelSpeeds": [ { "model": "underdog-saluki-27b-iq2-mix", "medianOutputTokensPerSec": 8.4, "turns": 10 } ], "lastTurn": { "model": "underdog-saluki-27b-iq2-mix", "inputTokens": 8147, "cachedInputTokens": 7958, "outputTokens": 78, "cost": 0, "durationMs": 1074275, "outputTokensPerSec": 8.499300573791094, "at": "2026-11-10T23:56:42.135Z" }, "lastUpdated": "2026-11-10T23:56:42.135Z" }, "anthropic-cli": { "quotaBuckets": [], "todayTurns": 0, "todayTokensIn": 0, "todayTokensCached": 0, "todayTokensOut": 0, "todayCost": 0, "totalTurns": 0, "totalTokensIn": 0, "totalTokensCached": 0, "totalTokensOut": 0, "totalCost": 0, "medianOutputTokensPerSec": null, "modelSpeeds": [], "lastTurn": null, "lastUpdated": "2026-11-11T00:02:59.641Z" } }, "source": "engine-log" }, "derived": { "durationMs": 5642367, "meanTokensPerSec": 8.36, "genTokensPerSec": 8.36, "genTokens": 39447, "promptTokensPerSec": 327.68, "peakRssMb": 14413, "peakGpuUtilPercent": 100, "billing": { "provider": "llama-cpp", "limitedQuota": [], "tokens": { "input": 172991, "output": 7731 } } } }, "host": { "hostname": "", "platform": "win32", "arch": "x64", "osRelease": "10.0.26300", "cpuCount": 24, "cpuModel": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", "totalRamGb": 31, "gpuModel": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", "framework": "llama-cpp", "frameworkBinary": "packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe", "frameworkVariant": "env", "frameworkBuild": { "buildNumber": "11960", "revision": "8642541b255bfb5dc39cd351a178be09013a55d6", "backend": "cuda", "cudaArchitectures": [ "60-virtual", "70-real", "75-real", "80-real", "86-real", "89-real", "90-real", "100-real", "120" ], "sidecarPresent": true } }, "timing": { "startedAt": "2026-11-10T22:28:51.913Z", "finishedAt": "2026-11-11T00:03:05.110Z", "timeToFirstArtifactMs": 1468768, "timeToLastArtifactWriteMs": 5644369, "timeToFirstTokenMs": 47564, "firstTurnTtftMs": 38167, "timeToFirstToolCallMs": 54301 }, "modes": { "mode": "auto", "engine": "llama-cpp", "resolvedModes": { "generalist": 0, "stepwise": 6 }, "steps": { "activated": 9, "activationEvents": 5, "completed": 3, "gateApprovals": 3, "gateRejections": 3, "redrives": 0, "perStep": [ { "ref": "regional-sales-rollup/1", "stepId": "compute", "activatedAt": "2026-11-10T22:56:43.295Z", "completedAt": null, "ms": null, "gateRejections": 3, "toolCalls": 8, "sessionIds": [ "c131086a-934c-42f7-b8f2-2f318a44d247" ] }, { "ref": "regional-sales-rollup/2", "stepId": "compute", "activatedAt": "2026-11-10T22:46:26.921Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 1, "sessionIds": [ "281dcdfc-ff61-4c13-aa78-7f846f2e0a4c" ] }, { "ref": "regional-sales-rollup/3", "stepId": "compute", "activatedAt": "2026-11-10T23:21:11.963Z", "completedAt": "2026-11-10T23:29:16.016Z", "ms": 484053, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "2cb64035-e6e2-47bb-9b9b-ed6c453f0bfe" ] }, { "ref": "regional-sales-rollup/3", "stepId": "write-summary", "activatedAt": "2026-11-10T23:29:16.017Z", "completedAt": "2026-11-10T23:56:42.093Z", "ms": 1646076, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "bd2a070f-bdb8-4b05-88c3-9d7391b5e259" ] }, { "ref": "regional-sales-rollup/4", "stepId": "compute", "activatedAt": "2026-11-10T23:21:12.324Z", "completedAt": "2026-11-10T23:38:42.862Z", "ms": 1050538, "gateRejections": 0, "toolCalls": 4, "sessionIds": [ "c47d1748-91f2-4d56-9fd5-9fb836986a32" ] }, { "ref": "regional-sales-rollup/4", "stepId": "write-summary", "activatedAt": "2026-11-10T23:38:42.864Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 2, "sessionIds": [ "aba6342d-07ec-4adf-b2dd-23b7fd06c050" ] }, { "ref": "regional-sales-rollup/3", "stepId": "finish", "activatedAt": "2026-11-10T23:56:42.094Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "shared/1", "stepId": "index", "activatedAt": "2026-11-10T22:28:57.389Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] }, { "ref": "default/1", "stepId": "oversight", "activatedAt": "2026-11-10T22:28:57.269Z", "completedAt": null, "ms": null, "gateRejections": 0, "toolCalls": 0, "sessionIds": [] } ], "medianStepMs": 1050538 }, "sessions": { "total": 9, "taskScoped": 6, "byGezel": { "margarida": 5, "mina": 3, "reza": 1 }, "perTask": { "regional-sales-rollup/2": 1, "regional-sales-rollup/3": 2, "regional-sales-rollup/1": 1, "regional-sales-rollup/4": 2 }, "reusedAcrossSteps": 0, "continuityReuses": 0, "continuityBreaks": 0, "sessionsPerStep": 0.6666666666666666, "maxMessages": 7, "maxContextFill": 0.5150299072265625, "resumeFailures": 0 }, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5150299072265625 }, "fanout": { "hosts": 0, "childrenSpawned": 0, "childrenCompleted": 0, "childrenFailed": 0, "barrierHolds": 0, "barrierReleases": 0, "barrierReleaseFailures": 0, "skipped": 0 }, "budget": { "taskBudgetSoft": 0, "taskBudgetHard": 0, "toolRepeatAborts": 0 } }, "codexReview": { "cellId": "r3-saluki-cuda-complex-work-craftbook-author-params", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-params", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "scenarioId": "craftbook-author-params", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-11T00:05:26.173Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/out/north-summary.md", "artifactBytes": 140, "artifactSha256": "c9ccfb95d076c62d7595b4ceb8151bb6a4e95b83c1212f81abb721b87d6e4ea8", "axisSource": "evals/src/craftbooks/authoring/author-params.ts; no declared judge axes", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "One revised recipe is invoked for both regions and produces accurate distinct summaries: north1298units/47847.95, south2270/83990.00, with correct top reps. Independent sums agree. YAML/regex repairs and duplicate task pairs cost94minutes/80calls; the gate passes when summaries exist, before both tasks finish.", "reviewBasis": "Read both summaries, both full CSVs, all three working-number artifacts and authoring/invocation/error receipts within20KiB per-file cap. Independently recomputed exact integer-cent totals and top reps. No changes to generated outputs, model retries or fixed rescoring.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/out/north-summary.md", "status": "notApplicable", "bytes": 140, "sha256": "c9ccfb95d076c62d7595b4ceb8151bb6a4e95b83c1212f81abb721b87d6e4ea8", "reviewedBytes": 140, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/out/south-summary.md", "status": "notApplicable", "bytes": 140, "sha256": "4679d8bf920313c80504983b8189dfc9a0fe9ade73c573d0676b399e56d3310c", "reviewedBytes": 140, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/data/north.csv", "status": "notApplicable", "bytes": 331, "sha256": "1d565713b89e9dc055719ef769bf6ba5bea1ff888d309aa34767a5231a0af9ba", "reviewedBytes": 331, "truncated": false, "role": "sourceData" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/data/south.csv", "status": "notApplicable", "bytes": 335, "sha256": "e68e801e3fc4700a2f4e7625ae8db017447668c1b36b0d19858bc309ac0be370", "reviewedBytes": 335, "truncated": false, "role": "sourceData" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/1/numbers.md", "status": "notApplicable", "bytes": 226, "sha256": "930c603b8ab05c4bae46dbf8a80e7899a7851e659fe372d7f79d80fb72582b72", "reviewedBytes": 226, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/3/numbers.md", "status": "notApplicable", "bytes": 454, "sha256": "320b444a78d89bd94a1eb5ea2156d2e645a67957c1bfa63e78963d5537ce2e14", "reviewedBytes": 454, "truncated": false, "role": "declaredOutput" }, { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/4/numbers.md", "status": "notApplicable", "bytes": 500, "sha256": "0315e9c3fd170b8258beed9a57a3a491e28f7ef022ad3a0c0db31245a455e6d5", "reviewedBytes": 500, "truncated": false, "role": "declaredOutput" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "offloadLines": [ "2026-11-10T22:28:59.456Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-10T22:29:02.569Z INFO [chat] [llama-server] launch {\"pid\":44808,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-11T00:03:04.386Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-44808-1794355384386\",\"pid\":44808,\"code\":null,\"signal\":\"SIGTERM\",\"expected\":true,\"expectedReason\":\"stop\",\"uptimeMs\":5641817,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeFailureLines": [], "maxObservedReleasedSlotTokens": 45541, "peakGpuMiB": 10646, "peakProcessRssMiB": 14413, "peakSystemRamMiB": 30878, "decodeTokensPerSec": 8.36, "prefillTokensPerSec": 327.68, "generatedTokens": 39447, "compaction": { "observable": true, "betweenTurn": 0, "midTurn": 1, "forceFit": 0, "compactStarts": 0, "compactFailed": 0, "firstTurnPrefixOver": 0, "loopHalts": 0, "maxContextFill": 0.5150299072265625 }, "note": "Actual task placement from launch/offload evidence; shared GPU memory and paging unmeasured. Whole-system memory includes other applications; memory-pressure cleanup labels alone do not prove OOM." } }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:07:31.378Z", "reviewId": "127", "cellId": "r3-saluki-cuda-complex-work-craftbook-author-params", "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "scenarioId": "craftbook-author-params", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-params", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "87a8ae817d437fa761fe323deb1e71d07586bba8ad21e090ee77c35af10f2bdd", "evidenceSha256": "a36312bbe5758d7265ebad76076625de22961ffcdb677427b8244684a0195fc8", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/out/north-summary.md", "bytes": 140, "sha256": "c9ccfb95d076c62d7595b4ceb8151bb6a4e95b83c1212f81abb721b87d6e4ea8", "shownBytes": 140, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/out/south-summary.md", "bytes": 140, "sha256": "4679d8bf920313c80504983b8189dfc9a0fe9ade73c573d0676b399e56d3310c", "shownBytes": 140, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/data/north.csv", "bytes": 331, "sha256": "1d565713b89e9dc055719ef769bf6ba5bea1ff888d309aa34767a5231a0af9ba", "shownBytes": 331, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/data/south.csv", "bytes": 335, "sha256": "e68e801e3fc4700a2f4e7625ae8db017447668c1b36b0d19858bc309ac0be370", "shownBytes": 335, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/1/numbers.md", "bytes": 226, "sha256": "930c603b8ab05c4bae46dbf8a80e7899a7851e659fe372d7f79d80fb72582b72", "shownBytes": 226, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/3/numbers.md", "bytes": 454, "sha256": "320b444a78d89bd94a1eb5ea2156d2e645a67957c1bfa63e78963d5537ce2e14", "shownBytes": 454, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/4/numbers.md", "bytes": 500, "sha256": "0315e9c3fd170b8258beed9a57a3a491e28f7ef022ad3a0c0db31245a455e6d5", "shownBytes": 500, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "No numeric axes are declared. Both summaries' totals match the CSVs: north is 1298 units, 47847.95 revenue and Cato Meijer; south is 2270 units, 83990.00 revenue and Nadia Roos. Intermediate numbers.md files exist for each run, which suggests a two-step recipe was run twice.", "limitations": "The craftbook itself was not shown, so the region parameter, the two-step structure, the order of authoring before running, and the use of craftbook_write cannot be verified. The disposition is runner-terminal, and the heading capitalization differs between the two summaries.", "meanScore": null, "durationMs": 14320, "costUsd": 0.01871, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3869, "cache_read_input_tokens": 700, "output_tokens": 316, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3869, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 316, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3869, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3869 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 316, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3869, "webSearchRequests": 0, "costUSD": 0.01871, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3869, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "\n# Trial postmortem - craftbook-author-params / underdog-saluki-27b-iq2-mix\n\n**Composite: 8.5 / 10** (band: ship-ready)\n**Outcome: success** (duration: 94m 13s; budget used: 50.2%)\n**Model aggregate eligibility: included (pass).**\n\n## Score breakdown\n\n| Axis | Score | Why |\n|---|---:|---|\n| Task completion (40%) | 10 / 10 | The scenario reached a successful terminal verdict. `facts.outcome.success: true` |\n| Output quality (25%) | 10 / 10 | The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1468768`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":323,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}` |\n| Process efficiency (20%) | 2.5 / 10 | Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.5019829544854162`; `facts.toolUse.totalToolCalls: 80`; `facts.timing.timeToFirstArtifactMs: 1468768`; `facts.outcome.durationMs: 5653197` |\n| Behavior soundness (15%) | 10 / 10 | No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1` |\n\nComposite = 0.4 * 10 + 0.25 * 10 + 0.2 * 2.5 + 0.15 * 10 = **8.5**.\n\n## Performance\n\nPerformance is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Peak RSS | 14413 MB | `facts.perf.process.peakRssMb` |\n| Peak GPU util | 100% | `facts.perf.gpu.peakUtilPercent` |\n| Peak GPU memory | 10646 MB / 12227 MB | `facts.perf.gpu.peakMemUsedMb`, `facts.perf.gpu.memoryModel` |\n| Peak system memory | 30878 MB / 31865 MB | `facts.perf.systemMemory.peakUsedMb`, `facts.perf.systemMemory.totalMb` |\n| Mean tokens/sec | 8.36 | `facts.perf.derived.meanTokensPerSec` |\n| Total tokens | 225830 + 39447 | `facts.perf.usage.totalInputTokens`, `facts.perf.usage.totalOutputTokens` |\n| Host | AMD Ryzen AI 9 HX 370 w/ Radeon 890M, 31 GB RAM, NVIDIA GeForce RTX 5070 Ti Laptop GPU | `facts.host.cpuModel`, `facts.host.totalRamGb`, `facts.host.gpuModel` |\n| Framework | llama-cpp via packages/app/native-bin/win32-x64-cuda/gezel-llama-server.exe | `facts.host.framework`, `facts.host.frameworkBinary` |\n| Engine build | env · build 11960 · 8642541b | `facts.host.frameworkVariant`, `facts.host.frameworkBuild` |\n\n## Continuity\n\nContinuity is reported separately and does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Generalist setting / resolved | auto / 0g 6s | `continuity.mode`, `continuity.resolvedModes` |\n| Steps activated → completed | 9 → 3 | `continuity.steps` |\n| Gate rejections / redrives | 3 / 0 | `continuity.steps` |\n| Median step wall-clock | 1051s | `continuity.steps.medianStepMs` |\n| Task sessions / sessions per step | 6 / 0.67 | `continuity.sessions` |\n| Sessions reused across steps | 0 (0 reuses, 0 breaks) | `continuity.sessions` |\n| Compactions | 0 between-turn / 1 mid-turn / 0 force-fit | `continuity.compaction` |\n| Max context fill | 52% | `continuity.compaction.maxContextFill` |\n| Compaction loop halts | 0 | `continuity.compaction.loopHalts` |\n| Fanout children spawned / completed / failed | 0 / 0 / 0 | `continuity.fanout` |\n| Fanout barrier holds / releases | 0 / 0 | `continuity.fanout` |\n| Task budget soft / hard trips | 0 / 0 | `continuity.budget` |\n| Tool-repeat aborts | 0 | `continuity.budget.toolRepeatAborts` |\n\n## Retrieval\n\nWhat reached the prompts from the indexes. Reported separately; it does not affect the capability composite.\n\n| Metric | Value | Source |\n|---|---:|---|\n| Arm | eval default | `retrieval.arm` |\n| Arm proof | applied | `retrievalArmProof` |\n| Policy modes / sources | — / — | `retrieval.policy` |\n| Turn probes → injections | 0 → 0 (0 sessions) | `retrieval.turn` |\n| Injected hits by source | — | `retrieval.turn.hitsBySource` |\n| Injected tokens | 0 | `retrieval.turn.injectedTokens` |\n| Rejected candidates | — | `retrieval.turn.rejected` |\n| Reference searches / tasks with references / items | 4 / 4 of 6 / 20 | `retrieval.references` |\n| Stamped messages | 0 | `retrieval.stampedMessages` |\n\n## Evidence map\n\n### Task completion - C_SUCCESS\n\nThe scenario reached a successful terminal verdict.\n\n- `facts.outcome.success: true`\n\n### Output quality - Q_GATE_COMPLETE\n\nThe scenario gate is complete.\n\n- `facts.outcome.success: true`\n- `facts.timing.timeToFirstArtifactMs: 1468768`\n- `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":323,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n\n### Process efficiency - E_ONE_SEVERE\n\nExactly one efficiency dimension reached the severe threshold.\n\n- `facts.outcome.budgetUsedFraction: 0.5019829544854162`\n- `facts.toolUse.totalToolCalls: 80`\n- `facts.timing.timeToFirstArtifactMs: 1468768`\n- `facts.outcome.durationMs: 5653197`\n\n### Behavior soundness - B_CLEAN_LOW_INTERVENTION\n\nNo red flags or missing roles were observed, with at most one intervention.\n\n- `facts.toolUse.redFlags: []`\n- `facts.team.missingExpectedRoles: []`\n- `facts.autoAnswer.total: 1`\n\n## Evidence-led observations\n\n- Completion: The scenario reached a successful terminal verdict. `facts.outcome.success: true`\n- Quality: The scenario gate is complete. `facts.outcome.success: true`; `facts.timing.timeToFirstArtifactMs: 1468768`; `facts.sniff.latest: {\"filePath\":\"craftbook-author-params\",\"bytes\":323,\"score\":7,\"scoreMax\":7,\"signals\":[],\"failReason\":null}`\n- Efficiency: Exactly one efficiency dimension reached the severe threshold. `facts.outcome.budgetUsedFraction: 0.5019829544854162`; `facts.toolUse.totalToolCalls: 80`; `facts.timing.timeToFirstArtifactMs: 1468768`; `facts.outcome.durationMs: 5653197`\n- Behavior: No red flags or missing roles were observed, with at most one intervention. `facts.toolUse.redFlags: []`; `facts.team.missingExpectedRoles: []`; `facts.autoAnswer.total: 1`\n\n## Raw evidence\n\n- Machine-readable fixed score: `score.json`.\n- Extracted observable facts: `facts.json`.\n- Terminal trial metadata and failure classification: `result.json`.\n\n\n## Enrichment needed\n\nThis deterministic report intentionally does not invent tactical or strategic recommendations. A follow-up evidence review should inspect `daemon.log`, model tuning, and relevant framework modules before adding:\n\n- Tactical fixes with a concrete file, field, and proposed value.\n- Strategic framework fixes tied to a reusable failure mode.\n- A cross-scenario generalization check and next experiment.\n\n\n## Inline advisory review — logical R3\n\nFixed **8.5/10**, pass; unchanged. Advisory **notApplicable**. One revised recipe is invoked for both regions and produces accurate distinct summaries: north1298units/47847.95, south2270/83990.00, with correct top reps. Independent sums agree. YAML/regex repairs and duplicate task pairs cost94minutes/80calls; the gate passes when summaries exist, before both tasks finish.\n\n\n\n## Positives\n\nIndependent integer-cent arithmetic matches both source CSVs exactly, including top-by-units Cato Meijer215/7955.25 and Nadia Roos410/15170.00. Region parameters reach input and output paths in the saved recipe. After correction, north and south use the same version1.0.1 with no edit between the two invocations, and separate task-scoped working files prevent their figures colliding.\n\n## Negatives and diagnosis\n\nThe first authoring attempt uses an unquoted interpolation token as a YAML pattern and is rejected. The accepted initial recipe contains unsupported inline regex flags, leading to three compute-step gate rejections; the subsequent repair needs another YAML quoting correction. The model cancels tasks1/2 and invokes3/4 on the repaired version. Grounding repeatedly rejects correctly derived totals and row counts until cited. A coordinator tries writing another task’s folder, receives the correct scope denial, then asks the simulated user for permission despite task workers already producing the required evidence. The harness auto-answers once. Facts.continuity.steps show north finish and south write-summary not completed when the scenario succeeds, so do not describe this as two fully completed workflows. The summaries lack source/period/currency labels, though these were not required or supplied. No numeric error was found.\n\n## Performance and fit\n\nDecode 8.36tok/s, prefill 327.68tok/s; 39447generated tokens. Peak GPU 10646/12227MiB, process RSS 14413MiB, whole-system RAM 30878/31865MiB. Context 65536, 1slot, f16; max observed released slot 45541. See advisory-tool-evidence.json for launch/exit/fault receipts. Shared GPU memory/paging are unmeasured; hybrid loading does not imply all-GPU residency or task competence.\n\n## Tactical experiment — not applied\n\nIn the existing authoring guidance, quote YAML regex strings and use the separate flags field rather than inline flag syntax. Keep the coordinator on status/routing work and let each task-scoped worker persist its own evidence. Include source paths in concise summaries without inventing a period or currency.\n\n## Strategic experiment — not applied\n\nValidate regex compilation before recipe acceptance in packages/service/src/http/routes/craftbooks.ts or shared gate validation, and distinguish derived numeric claims from unsupported numbers in grounding. In packages/service/src/tasks, expose task-owned progress clearly so the coordinator does not mistake its own write denial for a worker blocker. Preserve workspace authority boundaries. Future reporting should separate parameterized artifact correctness from task completion.\n\n## Historical and generalization check\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/facts.json): pass, fixed 8.5, 19.14minutes; original campaign environment.\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/facts.json): pass, fixed 8.5, 34.48minutes; original campaign environment.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/facts.json): fail, fixed 4.6, 7.47minutes; other historical baseline; contextual only.\n\nUnlike export-generalize, which labels unchanged invalid data clean, this attempt delivers independently correct aggregates for both datasets. Author-linear and data-wrangle also reconcile inputs correctly; AB analysis and records-intake fail to deliver. Across earlier same-scenario trials the fixed score measures a reusable input/output path, not proof of efficient or fully completed task execution.\n\nThe original/continuation environment boundary is explicit. No causal improvement claim, capability retry or source/tuning change. Proposals remain experiments requiring unrelated validation.\n\n## Recommended next experiment\n\nIn a fresh baseline, measure author-time regex validation and evidence-backed arithmetic across regional summaries, invoice totals and an unrelated review recipe; track correct outputs, task completion and avoidable repair calls separately.\n\nEvidence: facts.json, score.json, result.json, codex-inline-judge.json, advisory-tool-evidence.json, advisory-artifact-audit.json and historical-comparison.json.\n" }, { "cellId": "r2-saluki-cuda-complex-work-craftbook-author-fanout", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-author-fanout", "suite": "complex-work", "scenario": "craftbook-author-fanout", "repetition": 2, "runId": "2026-11-06-windows-saluki-native-0-1-57-fanout", "segment": "fanout", "disposition": "interrupted-without-runner-result", "eligible": false, "success": null, "result": null, "score": null, "facts": null, "metrics": null, "administrativeExclusion": { "observedAt": "2026-11-09T20:06:39.114Z", "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob", "cellId": "r2-saluki-cuda-complex-work-craftbook-author-fanout", "runDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob", "disposition": "interrupted-without-runner-result", "cause": "Unknown process interruption; user reports possible battery outage. No controlled shutdown, result.json, facts.json, score.json, metrics.json, or process-exit.json exists.", "lastPollAt": "2026-11-06T22:23:57.742Z", "lastDaemonAt": "2026-11-06T22:24:49.652Z", "staleStatus": "running", "processesConfirmedAbsent": [ 40532, 33296, 33840 ], "bootTimeObservation": "OS reported last boot 2026-10-30, so a reboot/battery shutdown is not established; suspension or process termination remain possible.", "fixedScore": null, "includedInModelAggregate": false, "classificationIsReviewerAdministrativeOnly": true, "retry": false, "retainedOriginalTrials": 85, "remainingUnattempted": 43, "standardResumeFinding": "all.ts has no same-trial checkpoint resume; score-trial requires result.json. Prepared round3 has no running/stopped cell or recorded rows and supports standard --execute after identity validation and preflight. Do not rerun either earlier one-shot launcher.", "evidenceSeal": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/campaign.json", "sha256": "1d4c4505ad0006962741b77c94c9564cf45492d18ebb6bfa719fc30237d677ae" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/summary.json", "sha256": "36fb467a92fba3648ab776c6f897adda6250fb0aaa06888938190986df1bb905" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/campaign.json", "sha256": "b83a73f4596bd46948ccf1cb27bcb6872edd037f8ecdccad139a2b965e284e89" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/summary.json", "sha256": "3e5432e7ef42baf9e841352fd93531ff49e6fc242b3d2f35809de984c210ec8f" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/process.json", "sha256": "665518f30bc681332e7d3ac478304bb6cf8ff399ba7f860c25b69271ce15a04a" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/status.json", "sha256": "649bd8d21997903c039d6bd0ff1b5020b72f5752dbde9511e488f9701caf07c2" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/log.txt", "sha256": "45ab5d6d7636f5fac7bac3056792f65cc52c92677bcda7b6e765c92695655b2c" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/daemon.log", "sha256": "9d6d340d90e2162e266a99e464cfc67b51b1bb62da9386d6ae0233c1d29e7cff" }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/recording/chat-events.jsonl", "sha256": "83f1210904315ed8183495cb31f3eb6a7245014797523a2a9d1c2d45ecb8e916" } ] }, "codexReview": { "cellId": "r2-saluki-cuda-complex-work-craftbook-author-fanout", "baselineCellId": null, "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06", "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob", "scenarioId": "craftbook-author-fanout", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-09T20:06:39.114Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/harbor-health.md", "artifactBytes": 465, "artifactSha256": "d7426541b3e86cfcbd8c7299e359cc06f1bf27fe9e714558212767ec80d77c5c", "axisSource": "evals/src/craftbooks/authoring/author-fanout.ts", "axes": [], "scoreAxes": {}, "axisJustifications": {}, "meanScore": null, "toolCalls": 96, "deliverables": [ { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/workspace/data/stores.json", "relativePath": "workspace/data/stores.json", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/data/stores.json", "bytes": 517, "sha256": "8bf728bad631198134c68817a9d08eaf08804c45fe27af05c1d8433c7e573e22", "status": "sourceFixture", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/data/stores.json" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/workspace/out/harbor-health.md", "relativePath": "workspace/out/harbor-health.md", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/harbor-health.md", "bytes": 465, "sha256": "d7426541b3e86cfcbd8c7299e359cc06f1bf27fe9e714558212767ec80d77c5c", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/harbor-health.md" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/workspace/out/linden-health.md", "relativePath": "workspace/out/linden-health.md", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/linden-health.md", "bytes": 439, "sha256": "8ef973d93c6c903ceef4059a0d0b632a5c5c0622ca604bda6e03e9905d7e2a87", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/linden-health.md" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/workspace/out/northside-health.md", "relativePath": "workspace/out/northside-health.md", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/northside-health.md", "bytes": 568, "sha256": "bfcc1c20556b9693d78755dae60f301a4450c62839a328e3c4478b8446f3f435", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/northside-health.md" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/workspace/out/milo-health.md", "relativePath": "workspace/out/milo-health.md", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/milo-health.md", "bytes": 415, "sha256": "45bb8c573b603063fe068c20b57da13f49044995952cc9005ffb138fe8560ca2", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/milo-health.md" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/workspace/craftbook-store-health-sweep.md", "relativePath": "workspace/craftbook-store-health-sweep.md", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/craftbook-store-health-sweep.md", "bytes": 1523, "sha256": "df900908211646571494432819ebec3f794c2aa78822a9555f14bf19ef4156b9", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/craftbook-store-health-sweep.md" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/artifacts/tasks/1/worklist.md", "relativePath": "artifacts/tasks/1/worklist.md", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/worklist.md", "bytes": 294, "sha256": "059540c626d17aa8c9f52dce2b865c5b10adde65e720a07997f0ffb252f08314", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/worklist.md" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/artifacts/tasks/1/manifest.md", "relativePath": "artifacts/tasks/1/manifest.md", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/manifest.md", "bytes": 180, "sha256": "dbff95f7ad5623fa4bc6b7e3082d83097c08b7c025b12f5f77a0ce34ccf865b8", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/manifest.md" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/artifacts/tasks/1/verdict.md", "relativePath": "artifacts/tasks/1/verdict.md", "bytes": 567, "status": "reviewedNoDeclaredAxes", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/verdict.md", "sha256": "3735875664cfa9b09eda3339bb7e7b9aa160f05dc4dd3cc801f59aa64dcafa69", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/verdict.md" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/projects/store-health-sweep/tasks/1/task.json", "relativePath": "tasks/1/task.json", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/tasks/1/task.json", "bytes": 17250, "sha256": "3d929187bbc6b79fe9aec65fb4d2fb1c2bbb64af48c0b466cf8651eb7a7c6d6e", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/tasks/1/task.json" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/craftbook-templates/st/store-health-sweep/manifest.json", "relativePath": "craftbook-templates/st/store-health-sweep/manifest.json", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/manifest.json", "bytes": 1239, "sha256": "e9f4072dc1cd176e8261006c7f624b5114b5e93034a449aa0c4252db11a2c960", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/manifest.json" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/craftbook-templates/st/store-health-sweep/versions/1.0.0/manifest.json", "relativePath": "craftbook-templates/st/store-health-sweep/versions/1.0.0/manifest.json", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/versions/1.0.0/manifest.json", "bytes": 10622, "sha256": "1ddd068a6da20d94c118e4975705ea4a322d2407b99a33e41ae7178a9c5b56e5", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/versions/1.0.0/manifest.json" }, { "originalPath": "/AppData/Local/Temp/gezel-eval-craftbook-author-fanout-nu3aKe/craftbook-templates/st/store-health-sweep/versions/1.0.0/about.md", "relativePath": "craftbook-templates/st/store-health-sweep/versions/1.0.0/about.md", "snapshot": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/versions/1.0.0/about.md", "bytes": 1011, "sha256": "6580bb27539016e63761ba25663f27655a036f003f8612ff15c051efd52539d6", "status": "reviewedNoDeclaredAxes", "unchangedFromPrevious": true, "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/versions/1.0.0/about.md" } ], "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "maxObservedReleasedSlotTokens": 48575, "nativeFailureLines": [], "compactionLines": [ "2026-11-06T20:50:13.319Z INFO [chat] [chat] deterministic mid-loop compaction for session fe7c9a6a (36588/65536 estimated tokens)", "2026-11-06T22:04:07.347Z INFO [chat] [chat] deterministic mid-loop compaction for session fe7c9a6a (45209/65536 estimated tokens)" ], "latestGeneration": "2026-11-06T22:24:49.652Z INFO [chat] [llama-server] 149.54.994.300 I slot print_timing: id 0 | task 63267 | n_gen = 191, tg = 7.76 t/s, tg_3s = 4.24 t/s", "sandboxProbe": "2026-11-06T19:54:49.576Z INFO [sandbox] [sandbox] windows-mxc available (tier=base-container, 340ms probe)", "observedAt": "2026-11-06T21:40:30.9527352Z", "gpuPointMiB": 10646, "gpuTotalMiB": 12227, "gpuUtilPointPercent": 27, "wholeSystemRamPointMiB": 28425, "wholeSystemRamTotalMiB": 31865, "nativeWorkingSetPointBytes": 12332523520, "sampleKind": "point-not-peak", "note": "Previous RAM/GPU values are point samples from November 6, not whole-trial peaks. No final metrics.json survived; shared GPU memory and paging unmeasured. No native/OOM/context failure recorded before abrupt logging cessation." }, "justification": "No declared axes. Four reports remain faithful to the supplied store data; one sequential task completed. No installed spawn or child tasks exist after six rejected fanout proposals. Processes ended without a runner result. Preserve this as an interrupted, unscored attempt, excluded from model pass-rate aggregates.", "reviewBasis": "All 13 saved files exactly match the prior reviewed hashes. Inspected the final 12 tool calls, three additional validation rejections, unchanged installed version/task state, and the abrupt log end. No generated code executed, no final fixed outcome invented.", "executionDisposition": "interrupted-without-runner-result", "fixedScore": null, "includedInModelAggregate": false, "interruptionEvidence": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-interruption-2026-11-09.json" }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:03:57.194Z", "reviewId": "086", "cellId": "r2-saluki-cuda-complex-work-craftbook-author-fanout", "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob", "scenarioId": "craftbook-author-fanout", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob", "baselineCellId": "r2-saluki-cuda-complex-work-craftbook-author-fanout", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06", "promptSha256": "7ce6bed90d027222d47462f32f4c1a7d69d17df1b74025c00fe4750912e8cd44", "evidenceSha256": "2530f28711e6e91a207315bc8da0bac0669ec6f58b1b39c13bed5530a67100fe", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/harbor-health.md", "bytes": 465, "sha256": "d7426541b3e86cfcbd8c7299e359cc06f1bf27fe9e714558212767ec80d77c5c", "shownBytes": 465, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/data/stores.json", "bytes": 517, "sha256": "8bf728bad631198134c68817a9d08eaf08804c45fe27af05c1d8433c7e573e22", "shownBytes": 517, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/linden-health.md", "bytes": 439, "sha256": "8ef973d93c6c903ceef4059a0d0b632a5c5c0622ca604bda6e03e9905d7e2a87", "shownBytes": 439, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/northside-health.md", "bytes": 568, "sha256": "bfcc1c20556b9693d78755dae60f301a4450c62839a328e3c4478b8446f3f435", "shownBytes": 568, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/milo-health.md", "bytes": 415, "sha256": "45bb8c573b603063fe068c20b57da13f49044995952cc9005ffb138fe8560ca2", "shownBytes": 415, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/craftbook-store-health-sweep.md", "bytes": 1523, "sha256": "df900908211646571494432819ebec3f794c2aa78822a9555f14bf19ef4156b9", "shownBytes": 1523, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/worklist.md", "bytes": 294, "sha256": "059540c626d17aa8c9f52dce2b865c5b10adde65e720a07997f0ffb252f08314", "shownBytes": 294, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/manifest.md", "bytes": 180, "sha256": "dbff95f7ad5623fa4bc6b7e3082d83097c08b7c025b12f5f77a0ce34ccf865b8", "shownBytes": 180, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/verdict.md", "bytes": 567, "sha256": "3735875664cfa9b09eda3339bb7e7b9aa160f05dc4dd3cc801f59aa64dcafa69", "shownBytes": 567, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/tasks/1/task.json", "bytes": 17250, "sha256": "3d929187bbc6b79fe9aec65fb4d2fb1c2bbb64af48c0b466cf8651eb7a7c6d6e", "shownBytes": 17250, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/manifest.json", "bytes": 1239, "sha256": "e9f4072dc1cd176e8261006c7f624b5114b5e93034a449aa0c4252db11a2c960", "shownBytes": 1239, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/versions/1.0.0/manifest.json", "bytes": 10622, "sha256": "1ddd068a6da20d94c118e4975705ea4a322d2407b99a33e41ae7178a9c5b56e5", "shownBytes": 10622, "truncated": false }, { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/versions/1.0.0/about.md", "bytes": 1011, "sha256": "6580bb27539016e63761ba25663f27655a036f003f8612ff15c051efd52539d6", "shownBytes": 1011, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "Run was interrupted with no runner result, so no final outcome is judged. Partial evidence: four per-store write-ups exist with numbers matching stores.json, and the task ran a craftbook with a list-driven loop step and all-PASS verdict. The workspace craftbook file differs from the executed one.", "limitations": "Evidence is source-only. It doesn't show craftbook_read/craftbook_write calls, authoring order, or whether the saved book was the one invoked. A workspace craftbook-*.md differs from the task's executed craftbook, and the interruption leaves final completion unverified.", "meanScore": null, "durationMs": 17575, "costUsd": 0.08011800000000001, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 19231, "cache_read_input_tokens": 700, "output_tokens": 312, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 19231, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 312, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 19231, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 19231 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 312, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 19231, "webSearchRequests": 0, "costUSD": 0.08011800000000001, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 19231, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "# Interrupted trial review — Saluki / craftbook-author-fanout\n\nThis attempt stopped without a runner result. It has **no fixed score** and is administratively excluded from model pass-rate aggregates. The user reported a possible battery outage; the process absence and abrupt log end confirm interruption, not its cause. The last daemon evidence is 2026-11-06T22:24:49.652Z; the last poll is 2026-11-06T22:23:57.742Z. The stale running status is preserved as raw evidence.\n\nFour report files remain byte-for-byte identical to the reviewed snapshot. Names, cities, issue counts and audit ages match the source; audit cadence, severity labels and effort estimates are author assumptions. Task 1 completed all four sequential stages, including a supported verification verdict. The installed recipe is still version 1.0.0, with no top-level spawn and no child tasks. The last observed gate was 6/8, which is a progress observation, not a terminal score.\n\nThe final 12 tool calls brought the total to 96. Three further proposals were rejected: two repeated the missing explicit read_artifact procedure errors, then a rewritten document omitted required step names and used the unsupported deliverable kind artifact. No new recipe was saved. Tool receipts and unchanged outputs are preserved under advisory-interruption-2026-11-09. Advisory status is notApplicable because the scenario declares no judge axes.\n\nNative context was 65,536 tokens, one slot, f16 KV; dense FFN placement remained 39 GPU / 25 CPU. Maximum observed released slot: 48575 tokens. Last generation sample: 2026-11-06T22:24:49.652Z INFO [chat] [llama-server] 149.54.994.300 I slot print_timing: id 0 | task 63267 | n_gen = 191, tg = 7.76 t/s, tg_3s = 4.24 t/s. There are 2 observed mid-loop compactions and 0 explicit native/OOM/context failure lines. Earlier point samples were 10,646/12,227 MiB GPU and 28,425/31,865 MiB whole-system RAM, with native working set 12,332,523,520 bytes. Final peak metrics were not written; shared GPU memory/paging are unmeasured.\n\n## Historical and unrelated comparisons\n\n- [underdog-saluki-27b-iq2-mix](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/facts.json): pass, fixed 8.1/10, 226.4 minutes.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556/facts.json): fail, fixed 3.5/10, 12.0 minutes.\n- [gemma4-12b-q4](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c/facts.json): fail, fixed 4.5/10, 8.7 minutes.\n\nOriginal Saluki R1 fanout passed at 8.1 after about 226 minutes. The current interruption at about 150 observed minutes cannot establish eventual success or failure. The original and continuation have an explicit GEZEL_MXC_EXEC environment boundary. Other-model/native history is contextual only. Parameterized authoring succeeded at 8.5 on R2; developer codemod rename also succeeded at 8.5. File creation and simple workflow competence therefore do not establish reliable complex fanout authoring.\n\n## Experiments only — none applied\n\n- Tactical: in packages/core/src/craftbook-doc.ts validation diagnostics, show one schema-valid read_artifact procedure example per unique defect, instead of repeating it for every consuming step. Measure repair turns and invalid rewrites in both batch fanout and unrelated document authoring; preserve acceptance rules.\n- Strategic: inspect the literal backtick-based read_artifact check in packages/core/src/schemas/craftbook.ts and the authoring documentation retrieval path. Evaluate semantic procedure validation and a schema-aware edit operation against valid and invalid recipes across unrelated tasks. Repeated 17–19 KiB proposals and irrelevant how_do_i retrievals motivate the experiment, not a weakened gate or scenario hint.\n\nNo model tuning, scenario prompt, gate, source code or previous fixed score was changed. The standard all.ts runner has no checkpoint continuation for this incomplete trial. The separate prepared 43-entry round3 scorecard remains unattempted; its frozen identity matches and it can start through its standard --execute path with preflight. This interrupted logical R2 attempt will not be silently repeated.\n" }, { "cellId": "r3-saluki-cuda-complex-work-craftbook-author-gate-script", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "suite": "complex-work", "scenario": "craftbook-author-gate-script", "repetition": 3, "runId": "2026-11-09-windows-saluki-native-0-1-57-round3", "segment": "round3", "disposition": "interrupted-without-runner-result", "eligible": false, "success": null, "result": null, "score": null, "facts": null, "metrics": null, "administrativeExclusion": { "asOf": "2026-11-11T14:33:39.382Z", "logicalCellId": "r3-saluki-cuda-complex-work-craftbook-author-gate-script", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294", "executionDisposition": "interrupted-without-runner-result", "includedInModelAggregate": false, "fixedScore": null, "lastDaemonEventAt": "2026-11-11T02:25:34.482Z", "lastRunnerLogAt": "2026-11-11T02:21:04.637Z", "hostInspection": { "at": "2026-11-11T14:27:18Z", "method": "Host Win32_Process inspection outside sandbox", "result": "No campaign launcher, eval Node process or llama server remained. Only unrelated tool runtimes and the inspection shell were observed." }, "cause": "Unconfirmed; no battery, power, OOM or paging attribution", "runnerExitFilePresent": false, "runnerStatePreserved": true, "unattemptedCellId": "r3-saluki-cuda-complex-work-craftbook-author-fanout", "snapshotFiles": [ { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/projects/inventory-health-check/workspace/data/inventory.json", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/workspace/data/inventory.json", "bytes": 1006, "sha256": "2a6f44cb0bb8d193c2abf527264e828fbf882e8a744180a786e784fe0688b439" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/projects/inventory-health-check/workspace/out/inventory-report.json", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/workspace/out/inventory-report.json", "bytes": 1479, "sha256": "875d410067ebd445fa86631d9ba7c0b171c32a9f2411174ca028bf0af43950be" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/projects/inventory-health-check/tasks/.next-id", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/tasks/.next-id", "bytes": 2, "sha256": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/projects/inventory-health-check/tasks/1/about.md", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/tasks/1/about.md", "bytes": 150, "sha256": "3c867fa42d31f4d570a598de66feafe3811a7565cc504650419129ead3075c19" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/projects/inventory-health-check/tasks/1/notes.jsonl", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/tasks/1/notes.jsonl", "bytes": 7518, "sha256": "f35b8bf699206a53d1e3713d67c3adcd4d664ee05f79c30978e1585126b4d08a" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/projects/inventory-health-check/tasks/1/task.json", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/tasks/1/task.json", "bytes": 14451, "sha256": "cd102aa962b3a216b1e4a1d5c1f93798a03d7de28898ddacbbfb7ecfea86ea26" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/craftbook-templates/in/inventory-health-report/manifest.json", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/craftbook-templates/in/inventory-health-report/manifest.json", "bytes": 793, "sha256": "67db4a076a840a0d0669b33b1ccecdcb0f65cba7c5fca4dd3a477a9052414f39" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/craftbook-templates/in/inventory-health-report/versions/1.0.0/about.md", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/craftbook-templates/in/inventory-health-report/versions/1.0.0/about.md", "bytes": 562, "sha256": "0c752e963323cb006706b5f6579ba57cb3e5a0b4dd2597b82cbbe10f57569dc8" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/craftbook-templates/in/inventory-health-report/versions/1.0.0/manifest.json", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/craftbook-templates/in/inventory-health-report/versions/1.0.0/manifest.json", "bytes": 5090, "sha256": "132d805a0f3e275ec9ae22360ca40c5b171151fae0381c0bd0659edcb6d275a9" }, { "source": "/AppData/Local/Temp/gezel-eval-craftbook-author-gate-script-njTbiU/craftbook-templates/in/inventory-health-report/versions/1.0.0/scripts/inventory-health-gate.ts", "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/craftbook-templates/in/inventory-health-report/versions/1.0.0/scripts/inventory-health-gate.ts", "bytes": 3636, "sha256": "bf5519238090215ccd010d863160505c2390ac76e81870170b9069b6dcd0d130" } ], "artifactAudit": { "totalItems": 10, "totalQuantity": 583, "totalValue": 7385.6, "exactItems": true, "health": "critical", "taskStatus": "paused", "gate": "Both installed revision and task snapshot retain illegal top-level return statements. Two recorded gate attempts failed ERR_INVALID_TYPESCRIPT_SYNTAX; no later successful gate or terminal task." }, "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "offloadLines": [ "2026-11-11T00:06:33.235Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-11T00:06:36.482Z INFO [chat] [llama-server] launch {\"pid\":49480,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}", "2026-11-11T00:10:33.808Z INFO [chat] [llama-server] launch {\"pid\":51964,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-11T00:10:33.168Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-49480-1794355833168\",\"pid\":49480,\"code\":3221226505,\"signal\":null,\"expected\":true,\"expectedReason\":\"memory-pressure\",\"uptimeMs\":236686,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "maxObservedReleasedSlotTokens": 50014, "latestGeneration": "2026-11-11T02:25:34.482Z INFO [chat] [llama-server] 135.00.633.322 I slot print_timing: id 0 | task 52660 | n_gen = 1824, tg = 7.26 t/s, tg_3s = 2.20 t/s", "finalMetricsAvailable": false, "note": "No complete-trial peaks or average throughput. Shared GPU memory and paging unmeasured. The earlier recovered endpoint/native-exit episode is separate from the unexplained final interruption." } }, "codexReview": { "cellId": "r3-saluki-cuda-complex-work-craftbook-author-gate-script", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294", "scenarioId": "craftbook-author-gate-script", "judgeProvider": "codex-inline", "judgeModel": "current-session", "status": "notApplicable", "provisional": false, "reviewedAt": "2026-11-11T14:33:39.382Z", "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/workspace/out/inventory-report.json", "artifactBytes": 1479, "artifactSha256": "875d410067ebd445fa86631d9ba7c0b171c32a9f2411174ca028bf0af43950be", "axisSource": "evals/src/craftbooks/authoring/gate-script.ts", "axes": [], "scoreAxes": {}, "meanScore": null, "justification": "No declared axes. The report matches all ten inventory items and totals 583 units/7385.60. The saved gate retains illegal top-level returns and the task remains paused. Logging ceased without a runner verdict and no eval processes remain. Preserve as interrupted and unscored; do not infer a task failure or battery cause.", "reviewBasis": "Read the declared output within the 20 KiB cap, seeded inventory, saved gate, task and diagnostic notes. Copied ten evidence files byte-for-byte. No generated gate execution or model retry.", "deliverables": [ { "artifactPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/workspace/out/inventory-report.json", "status": "notApplicable", "bytes": 1479, "reviewedBytes": 1479, "truncated": false, "sha256": "875d410067ebd445fa86631d9ba7c0b171c32a9f2411174ca028bf0af43950be", "role": "declaredOutput" } ], "toolCalls": 98, "health": { "allocatedContextTokens": 65536, "slots": 1, "kvCacheType": "f16", "gpuDenseFfnLayers": 39, "cpuDenseFfnLayers": 25, "offloadLines": [ "2026-11-11T00:06:33.235Z INFO [chat] [llama-cpp] offload plan (underdog-saluki-27b-iq2-mix): dense FFN of 39/64 layers fits VRAM (3.0 GiB of 4.5 GiB, non-FFN 2.9 GiB, reserves 5.5 GiB) — --n-cpu-ffn 25" ], "nativeLaunchLines": [ "2026-11-11T00:06:36.482Z INFO [chat] [llama-server] launch {\"pid\":49480,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}", "2026-11-11T00:10:33.808Z INFO [chat] [llama-server] launch {\"pid\":51964,\"command\":\"gezel-llama-server.exe\",\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "nativeExitLines": [ "2026-11-11T00:10:33.168Z INFO [chat] [llama-server] exit {\"incidentId\":\"native-49480-1794355833168\",\"pid\":49480,\"code\":3221226505,\"signal\":null,\"expected\":true,\"expectedReason\":\"memory-pressure\",\"uptimeMs\":236686,\"reachedReady\":true,\"nativeRelease\":\"0.1.57\",\"model\":\"underdog-saluki-27b-iq2-mix\",\"backend\":\"cuda\",\"upstreamRevision\":\"8642541b255bfb5dc39cd351a178be09013a55d6\",\"cudaArchitectures\":\"60-virtual;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120\",\"cudaToolkit\":\"12.8\",\"gpu\":\"NVIDIA GeForce RTX 5070 Ti Laptop GPU\",\"computeCapability\":\"12.0\",\"driverVersion\":\"610.88\",\"contextPerSlot\":65536,\"contextTotal\":65536,\"slots\":1,\"kvCacheType\":\"f16\",\"batchSize\":\"default\",\"ubatchSize\":\"512\",\"flashAttention\":\"default\",\"threads\":\"default\",\"reasoningBudgetTokens\":4096,\"reasoningBudgetSource\":\"catalog\",\"reasoningPreserve\":false,\"cudaGraphs\":\"default\"}" ], "maxObservedReleasedSlotTokens": 50014, "latestGeneration": "2026-11-11T02:25:34.482Z INFO [chat] [llama-server] 135.00.633.322 I slot print_timing: id 0 | task 52660 | n_gen = 1824, tg = 7.26 t/s, tg_3s = 2.20 t/s", "finalMetricsAvailable": false, "note": "No complete-trial peaks or average throughput. Shared GPU memory and paging unmeasured. The earlier recovered endpoint/native-exit episode is separate from the unexplained final interruption." }, "executionDisposition": "interrupted-without-runner-result", "fixedScore": null, "includedInModelAggregate": false, "interruptionEvidence": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3-interruption-2026-11-11.json" }, "sonnetReview": { "judgeProvider": "anthropic-cli", "judgeModel": "claude-sonnet-5-5", "actualModelIds": [ "claude-sonnet-5-5" ], "cliVersion": "2.1.260", "reviewedAt": "2026-11-11T17:07:39.326Z", "reviewId": "128", "cellId": "r3-saluki-cuda-complex-work-craftbook-author-gate-script", "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294", "scenarioId": "craftbook-author-gate-script", "status": "notApplicable", "axes": [], "trialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294", "baselineCellId": "r1-saluki-cuda-complex-work-craftbook-author-gate-script", "comparisonBaseline": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3", "promptSha256": "7d1632881b5df62f93ec0a8c0e21ac245cac5a450041bed7045fb5ecefaad8cc", "evidenceSha256": "258bbc330aaf91e671aa3db57fa6eeb9b08e5c1e2db1a4725b17329db62b8bc8", "artifactFiles": [ { "path": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/workspace/out/inventory-report.json", "bytes": 1479, "sha256": "875d410067ebd445fa86631d9ba7c0b171c32a9f2411174ca028bf0af43950be", "shownBytes": 1479, "truncated": false } ], "contextFiles": [], "scoreAxes": {}, "justification": "The run was interrupted with no runner result and no declared axes. The only artifact, out/inventory-report.json, is well-formed. It has totals, out-of-stock, negative and low-stock lists, and a health field. It was not checked against the input.", "limitations": "The craftbook, its inline gate script, the verification step, and the craftbook_write and invoke_craftbook calls are not in the evidence. The report's origin is unknown: the crew may have built it or it may have been written by hand. The input file was not shown, so accuracy is unverified.", "meanScore": null, "durationMs": 14173, "costUsd": 0.015166, "usage": { "input_tokens": 2, "cache_creation_input_tokens": 3043, "cache_read_input_tokens": 700, "output_tokens": 292, "output_tokens_details": { "thinking_tokens": 0 }, "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 0 }, "service_tier": "standard", "cache_creation": { "ephemeral_1h_input_tokens": 3043, "ephemeral_5m_input_tokens": 0 }, "inference_geo": "global", "iterations": [ { "input_tokens": 2, "output_tokens": 292, "cache_read_input_tokens": 700, "cache_creation_input_tokens": 3043, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3043 }, "type": "message" } ], "speed": "standard", "fallback_credit": null }, "modelUsage": { "claude-sonnet-5-5": { "inputTokens": 2, "outputTokens": 292, "cacheReadInputTokens": 700, "cacheCreationInputTokens": 3043, "webSearchRequests": 0, "costUSD": 0.015166, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "cacheCreationEphemeral5mInputTokens": 0, "cacheCreationEphemeral1hInputTokens": 3043, "canonicalModel": "claude-sonnet-5-5", "provider": "firstParty", "costBasis": "list" } } }, "postmortem": "# Interrupted trial — craftbook-author-gate-script / Saluki\n\nNo fixed score or terminal outcome exists. Administrative disposition: interrupted; excluded from model aggregates. Last native event: 2026-11-11T02:25:34.482Z. Host inspection later found no campaign or native processes. Cause unconfirmed. The runner's stale running status is preserved.\n\n## Advisory artifact review\n\nNo declared axes. The report matches all ten inventory items and totals 583 units/7385.60. The saved gate retains illegal top-level returns and the task remains paused. Logging ceased without a runner verdict and no eval processes remain. Preserve as interrupted and unscored; do not infer a task failure or battery cause.\n\nNo declared judge axes; status notApplicable. Source and report independently agree on 10 items, 583 units and 7385.60 value; item values and flags match. The task remains paused on build. Task notes record two ERR_INVALID_TYPESCRIPT_SYNTAX failures from top-level return statements. The later installed script fixes gateResult arguments but still retains top-level returns; the task snapshot remains an older revision. These are partial-work observations, not a final model failure.\n\n## Performance\n\n65536 tokens, one slot, f16; dense FFN 39 GPU / 25 CPU. One endpoint/nonzero-exit episode at 00:10:33Z recovered automatically; generation continued until 02:25:34Z. No final metrics survived, so no whole-trial peak or mean is asserted. Shared GPU memory and paging were not measured.\n\n## Historical and generalization check\n\n- underdog-saluki-27b-iq2-mix: fixed 5.2, excluded, original; [facts](evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/facts.json).\n- underdog-saluki-27b-iq2-mix: fixed 8.1, eligible, original; [facts](evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/facts.json).\n- gemma4-12b-q4: fixed 4.5, eligible, other-historical; [facts](evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/facts.json).\n\nOriginal R1 passed at 8.1 despite no successful custom gate and a 10.00 arithmetic error; R2 was infrastructure-excluded despite a correct report. This interrupted report is accurate but still does not demonstrate a working custom gate. Unrelated round3 data-wrangle and bug-fix-tdd have genuine exact-output/test evidence, while author-params demonstrates accurate numeric output with repair overhead. Environment boundaries prevent a causal improvement claim.\n\n## Tactical experiment — not applied\n\nIn the catalog developer role about.md, add one general instruction to validate a saved script as an ES module before task invocation and keep early returns inside an async function. Measure syntax-error recurrence and calls across gate authoring and unrelated TDD; do not include scenario names or fixture values in the prompt.\n\n## Strategic experiment — not applied\n\nIn packages/service/src/fs/craftbook-store.ts and packages/script-runtime/src/compile.ts, return source-position syntax diagnostics before persisting a script; expose task-snapshot versus installed-version differences during repair. Preserve task history. Validate across custom gates and unrelated parameterized recipes.\n\nEvidence: codex-inline-judge.json, advisory-artifact-audit.json, advisory-tool-evidence.json, historical-comparison.json, and advisory-interruption-2026-11-11/. No existing scores, facts, result or runner journals were changed.\n" } ], "unattempted": [ "r3-saluki-cuda-complex-work-craftbook-author-fanout" ], "historicalComparisons": { "asOf": "2026-11-11T14:33:39.382Z", "caveat": "Previous available scored trials by start time, across any model. Other models and environments are contextual, not controlled comparisons. Interrupted attempts are not scored.", "scenarios": [ { "scenario": "tictactoe", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-tictactoe\\tictactoe\\tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl", "priorThree": [ { "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T09:50:34.103Z", "success": true, "composite": 8.8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/postmortem.md" }, { "trialId": "tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T00:05:25.966Z", "success": true, "composite": 8.8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/postmortem.md" }, { "trialId": "tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T07:06:54.918Z", "success": true, "composite": 10, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tictactoe/tictactoe-gemma4-12b-q4-2026-10-28T07-06-54-917Z-ly7f/postmortem.md" } ] }, { "scenario": "petshop", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-petshop\\petshop\\petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy", "priorThree": [ { "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T09:59:21.726Z", "success": true, "composite": 7, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/postmortem.md" }, { "trialId": "petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T00:16:26.052Z", "success": true, "composite": 7.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/postmortem.md" }, { "trialId": "petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T07:13:04.371Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/petshop/petshop-gemma4-12b-q4-2026-10-28T07-13-04-369Z-lsz5/postmortem.md" } ] }, { "scenario": "tankcombat", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-tankcombat\\tankcombat\\tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8", "priorThree": [ { "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T11:05:34.375Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/postmortem.md" }, { "trialId": "tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T01:40:41.738Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/postmortem.md" }, { "trialId": "tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T07:17:31.541Z", "success": true, "composite": 10, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/tankcombat/tankcombat-gemma4-12b-q4-2026-10-28T07-17-31-539Z-3b9p/postmortem.md" } ] }, { "scenario": "schema-migration", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-schema-migration\\schema-migration\\schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203", "priorThree": [ { "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T11:15:55.019Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/postmortem.md" }, { "trialId": "schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T02:13:44.134Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/postmortem.md" }, { "trialId": "schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T07:23:06.185Z", "success": true, "composite": 9, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/schema-migration/schema-migration-gemma4-12b-q4-2026-10-28T07-23-06-184Z-j228/postmortem.md" } ] }, { "scenario": "failing-tests-spec", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-failing-tests-spec\\failing-tests-spec\\failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve", "priorThree": [ { "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T11:27:25.397Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/postmortem.md" }, { "trialId": "failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T02:44:03.486Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/postmortem.md" }, { "trialId": "failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T07:27:42.842Z", "success": false, "composite": 6.1, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/failing-tests-spec/failing-tests-spec-gemma4-12b-q4-2026-10-28T07-27-42-841Z-h3vh/postmortem.md" } ] }, { "scenario": "symptom-debug", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-symptom-debug\\symptom-debug\\symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn", "priorThree": [ { "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T11:31:24.505Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/postmortem.md" }, { "trialId": "symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T03:05:54.028Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/postmortem.md" }, { "trialId": "symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T07:31:39.415Z", "success": true, "composite": 10, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/symptom-debug/symptom-debug-gemma4-12b-q4-2026-10-28T07-31-39-414Z-ivlj/postmortem.md" } ] }, { "scenario": "data-wrangle", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-data-wrangle\\data-wrangle\\data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0", "priorThree": [ { "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T11:37:37.261Z", "success": false, "composite": 4.3, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/postmortem.md" }, { "trialId": "data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T03:15:59.498Z", "success": false, "composite": 3, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/postmortem.md" }, { "trialId": "data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T07:48:52.699Z", "success": false, "composite": 3.4, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/data-wrangle/data-wrangle-gemma4-12b-q4-2026-10-28T07-48-52-697Z-i55l/postmortem.md" } ] }, { "scenario": "incident-postmortem", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-incident-postmortem\\incident-postmortem\\incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj", "priorThree": [ { "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T13:26:14.308Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/postmortem.md" }, { "trialId": "incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T04:15:17.174Z", "success": false, "composite": 3.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/postmortem.md" }, { "trialId": "incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T08:01:43.451Z", "success": true, "composite": 10, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/incident-postmortem/incident-postmortem-gemma4-12b-q4-2026-10-28T08-01-43-450Z-6pp4/postmortem.md" } ] }, { "scenario": "ops-runbook-anomaly", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-core-ops-runbook-anomaly\\ops-runbook-anomaly\\ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2", "priorThree": [ { "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T13:43:51.253Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/postmortem.md" }, { "trialId": "ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T04:51:33.588Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/postmortem.md" }, { "trialId": "ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T08:06:28.642Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/core/ops-runbook-anomaly/ops-runbook-anomaly-gemma4-12b-q4-2026-10-28T08-06-28-641Z-nmab/postmortem.md" } ] }, { "scenario": "plan-and-estimate", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-plan-and-estimate\\plan-and-estimate\\plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o", "priorThree": [ { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-09T22:23:01.779Z", "success": true, "composite": 10, "eligible": true, "baseline": "continuation", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/postmortem.md" }, { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T16:17:40.855Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/postmortem.md" }, { "trialId": "plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T13:52:18.464Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/postmortem.md" } ] }, { "scenario": "conflict-synthesis", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-conflict-synthesis\\conflict-synthesis\\conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu", "priorThree": [ { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-09T22:28:21.715Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "continuation", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/postmortem.md" }, { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T17:48:58.019Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/postmortem.md" }, { "trialId": "conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T13:58:08.650Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/postmortem.md" } ] }, { "scenario": "constrained-comms", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-constrained-comms\\constrained-comms\\constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7", "priorThree": [ { "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T14:19:48.931Z", "success": false, "composite": 6, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/postmortem.md" }, { "trialId": "constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T06:15:01.379Z", "success": false, "composite": 6, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/postmortem.md" }, { "trialId": "constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T08:12:20.962Z", "success": true, "composite": 10, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/constrained-comms/constrained-comms-gemma4-12b-q4-2026-10-28T08-12-20-961Z-hmho/postmortem.md" } ] }, { "scenario": "records-intake", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-records-intake\\records-intake\\records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe", "priorThree": [ { "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T14:25:48.978Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/postmortem.md" }, { "trialId": "records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T06:25:47.186Z", "success": false, "composite": 2, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/postmortem.md" }, { "trialId": "records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T08:17:28.810Z", "success": true, "composite": 10, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/records-intake/records-intake-gemma4-12b-q4-2026-10-28T08-17-28-809Z-iaqu/postmortem.md" } ] }, { "scenario": "meeting-followup", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-meeting-followup\\meeting-followup\\meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3", "priorThree": [ { "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T16:23:32.329Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/postmortem.md" }, { "trialId": "meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T07:31:51.779Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/postmortem.md" }, { "trialId": "meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T08:26:39.548Z", "success": true, "composite": 10, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/meeting-followup/meeting-followup-gemma4-12b-q4-2026-10-28T08-26-39-547Z-d7u7/postmortem.md" } ] }, { "scenario": "craftbook-spreadsheet-model", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-spreadsheet-model\\craftbook-spreadsheet-model\\craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc", "priorThree": [ { "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T16:31:03.221Z", "success": true, "composite": 8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/postmortem.md" }, { "trialId": "craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T07:47:54.586Z", "success": false, "composite": 4.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/postmortem.md" }, { "trialId": "craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T08:34:42.001Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-spreadsheet-model/craftbook-spreadsheet-model-gemma4-12b-q4-2026-10-28T08-34-41-999Z-ln47/postmortem.md" } ] }, { "scenario": "docblocks-theme-roundtrip", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-docblocks-theme-roundtrip\\docblocks-theme-roundtrip\\docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls", "priorThree": [ { "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T17:58:27.610Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/postmortem.md" }, { "trialId": "docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T09:39:57.682Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/postmortem.md" }, { "trialId": "docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T08:55:40.884Z", "success": false, "composite": 5.9, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/docblocks-theme-roundtrip/docblocks-theme-roundtrip-gemma4-12b-q4-2026-10-28T08-55-40-883Z-qps3/postmortem.md" } ] }, { "scenario": "craftbook-powerpoint-deck", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-powerpoint-deck\\craftbook-powerpoint-deck\\craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b", "priorThree": [ { "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T18:07:11.971Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/postmortem.md" }, { "trialId": "craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T10:01:30.181Z", "success": false, "composite": 4.6, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/postmortem.md" }, { "trialId": "craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T09:09:42.723Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-powerpoint-deck/craftbook-powerpoint-deck-gemma4-12b-q4-2026-10-28T09-09-42-722Z-jjgw/postmortem.md" } ] }, { "scenario": "wikipedia-research-brief", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-wikipedia-research-brief\\wikipedia-research-brief\\wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl", "priorThree": [ { "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T18:42:31.351Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/postmortem.md" }, { "trialId": "wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T11:17:39.778Z", "success": false, "composite": 3, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/postmortem.md" }, { "trialId": "wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T09:17:59.439Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/wikipedia-research-brief/wikipedia-research-brief-gemma4-12b-q4-2026-10-28T09-17-59-438Z-e4fw/postmortem.md" } ] }, { "scenario": "craftbook-week-plan", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-week-plan\\craftbook-week-plan\\craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz", "priorThree": [ { "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T19:02:34.586Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/postmortem.md" }, { "trialId": "craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T12:04:36.262Z", "success": true, "composite": 9, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/postmortem.md" }, { "trialId": "craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T09:38:04.737Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-week-plan/craftbook-week-plan-gemma4-12b-q4-2026-10-28T09-38-04-736Z-zwng/postmortem.md" } ] }, { "scenario": "craftbook-ab-test-readout", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-ab-test-readout\\craftbook-ab-test-readout\\craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1", "priorThree": [ { "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T19:37:35.763Z", "success": true, "composite": 8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/postmortem.md" }, { "trialId": "craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T13:40:24.958Z", "success": true, "composite": 9, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/postmortem.md" }, { "trialId": "craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T11:08:52.041Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-ab-test-readout/craftbook-ab-test-readout-gemma4-12b-q4-2026-10-28T11-08-52-040Z-myv7/postmortem.md" } ] }, { "scenario": "craftbook-annotated-bibliography", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-annotated-bibliography\\craftbook-annotated-bibliography\\craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd", "priorThree": [ { "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T21:35:05.920Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/postmortem.md" }, { "trialId": "craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T15:04:12.087Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/postmortem.md" }, { "trialId": "craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T11:19:33.788Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-annotated-bibliography/craftbook-annotated-bibliography-gemma4-12b-q4-2026-10-28T11-19-33-787Z-29rq/postmortem.md" } ] }, { "scenario": "craftbook-research-to-document", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-productivity-craftbook-research-to-document\\craftbook-research-to-document\\craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e", "priorThree": [ { "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T21:53:08.914Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/postmortem.md" }, { "trialId": "craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T15:28:35.769Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/postmortem.md" }, { "trialId": "craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T11:28:52.534Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/productivity/craftbook-research-to-document/craftbook-research-to-document-gemma4-12b-q4-2026-10-28T11-28-52-533Z-hvyn/postmortem.md" } ] }, { "scenario": "fix-squisq-bugs", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-fix-squisq-bugs\\fix-squisq-bugs\\fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t", "priorThree": [ { "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T22:21:46.182Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/postmortem.md" }, { "trialId": "fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T15:51:11.511Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/postmortem.md" }, { "trialId": "fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T11:35:22.811Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/fix-squisq-bugs/fix-squisq-bugs-gemma4-12b-q4-2026-10-28T11-35-22-810Z-cxxy/postmortem.md" } ] }, { "scenario": "dev-craftbook-routing", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-dev-craftbook-routing\\dev-craftbook-routing\\dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb", "priorThree": [ { "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T22:32:46.674Z", "success": true, "composite": 8.8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/postmortem.md" }, { "trialId": "dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T16:08:27.564Z", "success": true, "composite": 8.8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/postmortem.md" }, { "trialId": "dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T11:53:03.728Z", "success": true, "composite": 8.8, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/dev-craftbook-routing/dev-craftbook-routing-gemma4-12b-q4-2026-10-28T11-53-03-726Z-jncy/postmortem.md" } ] }, { "scenario": "interface-contract", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-interface-contract\\interface-contract\\interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj", "priorThree": [ { "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T22:36:51.369Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/postmortem.md" }, { "trialId": "interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T16:12:00.767Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/postmortem.md" }, { "trialId": "interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T11:56:04.293Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/interface-contract/interface-contract-gemma4-12b-q4-2026-10-28T11-56-04-292Z-qgfj/postmortem.md" } ] }, { "scenario": "craftbook-code-review", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-code-review\\craftbook-code-review\\craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb", "priorThree": [ { "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T22:45:20.932Z", "success": true, "composite": 10, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/postmortem.md" }, { "trialId": "craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T16:17:39.750Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/postmortem.md" }, { "trialId": "craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T12:00:41.768Z", "success": true, "composite": 9.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-code-review/craftbook-code-review-gemma4-12b-q4-2026-10-28T12-00-41-767Z-d2u6/postmortem.md" } ] }, { "scenario": "craftbook-api-contract-review", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-api-contract-review\\craftbook-api-contract-review\\craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja", "priorThree": [ { "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T23:05:09.446Z", "success": true, "composite": 9, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/postmortem.md" }, { "trialId": "craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T16:46:58.461Z", "success": true, "composite": 9, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/postmortem.md" }, { "trialId": "craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T12:12:41.780Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-api-contract-review/craftbook-api-contract-review-gemma4-12b-q4-2026-10-28T12-12-41-779Z-xbgc/postmortem.md" } ] }, { "scenario": "craftbook-deep-security-review", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-deep-security-review\\craftbook-deep-security-review\\craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava", "priorThree": [ { "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T00:11:01.048Z", "success": true, "composite": 9, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/postmortem.md" }, { "trialId": "craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T17:45:22.804Z", "success": false, "composite": 5.3, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/postmortem.md" }, { "trialId": "craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T12:32:10.795Z", "success": false, "composite": 6.3, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-deep-security-review/craftbook-deep-security-review-gemma4-12b-q4-2026-10-28T12-32-10-794Z-4cl4/postmortem.md" } ] }, { "scenario": "large-pr-review", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-large-pr-review\\large-pr-review\\large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29", "priorThree": [ { "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T00:47:11.238Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/postmortem.md" }, { "trialId": "large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T18:07:39.733Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/postmortem.md" }, { "trialId": "large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T13:11:28.880Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/large-pr-review/large-pr-review-gemma4-12b-q4-2026-10-28T13-11-28-879Z-tf5i/postmortem.md" } ] }, { "scenario": "craftbook-codemod-sweep", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-codemod-sweep\\craftbook-codemod-sweep\\craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3", "priorThree": [ { "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T02:32:28.349Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/postmortem.md" }, { "trialId": "craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T19:37:25.767Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/postmortem.md" }, { "trialId": "craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T13:58:57.646Z", "success": false, "composite": 5.3, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-codemod-sweep/craftbook-codemod-sweep-gemma4-12b-q4-2026-10-28T13-58-57-645Z-pndu/postmortem.md" } ] }, { "scenario": "craftbook-refactor-module", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-refactor-module\\craftbook-refactor-module\\craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4", "priorThree": [ { "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T03:07:31.672Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/postmortem.md" }, { "trialId": "craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T20:08:16.627Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/postmortem.md" }, { "trialId": "craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T14:41:02.825Z", "success": false, "composite": 5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-refactor-module/craftbook-refactor-module-gemma4-12b-q4-2026-10-28T14-41-02-825Z-6owe/postmortem.md" } ] }, { "scenario": "craftbook-bug-fix-tdd", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-developer-craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd\\craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty", "priorThree": [ { "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T04:12:52.340Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/postmortem.md" }, { "trialId": "craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T21:11:12.352Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/postmortem.md" }, { "trialId": "craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T15:57:36.072Z", "success": false, "composite": 5.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/developer/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-gemma4-12b-q4-2026-10-28T15-57-36-072Z-4yic/postmortem.md" } ] }, { "scenario": "craftbook-find-vs-create", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-find-vs-create\\craftbook-find-vs-create\\craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq", "priorThree": [ { "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T04:40:51.156Z", "success": true, "composite": 8.8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/postmortem.md" }, { "trialId": "craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T21:37:06.557Z", "success": true, "composite": 8.8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/postmortem.md" }, { "trialId": "craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T16:44:25.443Z", "success": true, "composite": 8.8, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-find-vs-create/craftbook-find-vs-create-gemma4-12b-q4-2026-10-28T16-44-25-442Z-mese/postmortem.md" } ] }, { "scenario": "craftbook-edit-midtask", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-edit-midtask\\craftbook-edit-midtask\\craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67", "priorThree": [ { "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T04:45:07.149Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/postmortem.md" }, { "trialId": "craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T21:41:20.385Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/postmortem.md" }, { "trialId": "craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T16:50:49.990Z", "success": false, "composite": 4.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-edit-midtask/craftbook-edit-midtask-gemma4-12b-q4-2026-10-28T16-50-49-989Z-llwv/postmortem.md" } ] }, { "scenario": "craftbook-invoice-run", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-invoice-run\\craftbook-invoice-run\\craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml", "priorThree": [ { "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T05:51:45.283Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/postmortem.md" }, { "trialId": "craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-04T23:03:24.958Z", "success": false, "composite": 5.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/postmortem.md" }, { "trialId": "craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T17:04:35.828Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-invoice-run/craftbook-invoice-run-gemma4-12b-q4-2026-10-28T17-04-35-827Z-edom/postmortem.md" } ] }, { "scenario": "craftbook-author-linear", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-author-linear\\craftbook-author-linear\\craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm", "priorThree": [ { "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T06:22:58.630Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/postmortem.md" }, { "trialId": "craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T00:05:17.290Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/postmortem.md" }, { "trialId": "craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T17:17:47.951Z", "success": false, "composite": 5.4, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-linear/craftbook-author-linear-gemma4-12b-q4-2026-10-28T17-17-47-946Z-6z6y/postmortem.md" } ] }, { "scenario": "craftbook-route-multi", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-route-multi\\craftbook-route-multi\\craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr", "priorThree": [ { "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T07:33:08.380Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/postmortem.md" }, { "trialId": "craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T00:50:32.682Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/postmortem.md" }, { "trialId": "craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T17:51:27.010Z", "success": false, "composite": 5.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-route-multi/craftbook-route-multi-gemma4-12b-q4-2026-10-28T17-51-27-009Z-tmy5/postmortem.md" } ] }, { "scenario": "craftbook-export-generalize", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-export-generalize\\craftbook-export-generalize\\craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi", "priorThree": [ { "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T08:10:28.853Z", "success": false, "composite": 4.8, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/postmortem.md" }, { "trialId": "craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T01:25:52.409Z", "success": false, "composite": 2.4, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/postmortem.md" }, { "trialId": "craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T18:15:29.421Z", "success": false, "composite": 3.3, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-export-generalize/craftbook-export-generalize-gemma4-12b-q4-2026-10-28T18-15-29-420Z-pcwr/postmortem.md" } ] }, { "scenario": "craftbook-author-params", "currentTrialDir": "evals\\runs\\saluki-native-0.1.57-2026-11-03\\continuation-2026-11-06\\round3\\r1-saluki-cuda-complex-work-craftbook-author-params\\craftbook-author-params\\craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f", "priorThree": [ { "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T14:54:26.200Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/postmortem.md" }, { "trialId": "craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T02:38:35.029Z", "success": true, "composite": 8.5, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/postmortem.md" }, { "trialId": "craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T18:44:17.004Z", "success": false, "composite": 4.6, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-params/craftbook-author-params-gemma4-12b-q4-2026-10-28T18-44-17-003Z-8ize/postmortem.md" } ] }, { "scenario": "craftbook-author-gate-script", "currentTrialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294", "priorThree": [ { "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-06T15:16:17.979Z", "success": false, "composite": 5.2, "eligible": false, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/postmortem.md" }, { "trialId": "craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T03:15:52.590Z", "success": true, "composite": 8.1, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/postmortem.md" }, { "trialId": "craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T19:07:37.998Z", "success": false, "composite": 4.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-gate-script/craftbook-author-gate-script-gemma4-12b-q4-2026-10-28T19-07-37-997Z-rtu4/postmortem.md" } ] }, { "scenario": "craftbook-author-fanout", "currentTrialDir": "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob", "priorThree": [ { "trialId": "craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04", "modelId": "underdog-saluki-27b-iq2-mix", "startedAt": "2026-11-05T06:01:02.972Z", "success": true, "composite": 8.1, "eligible": true, "baseline": "original", "factsPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/facts.json", "postmortemPath": "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/postmortem.md" }, { "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T20:27:32.377Z", "success": false, "composite": 3.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-27-32-376Z-r556/postmortem.md" }, { "trialId": "craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c", "modelId": "gemma4-12b-q4", "startedAt": "2026-10-28T20:18:48.149Z", "success": false, "composite": 4.5, "eligible": true, "baseline": "other-historical", "factsPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c/facts.json", "postmortemPath": "evals/runs/scorecard-2026-10-27-windows-gemma4-12b-native-0-1-54/gemma4-12b-q4/complex-work/craftbook-author-fanout/craftbook-author-fanout-gemma4-12b-q4-2026-10-28T20-18-48-147Z-vl8c/postmortem.md" } ], "currentTrialDisposition": "interrupted R2, no score; R3 unattempted" } ] }, "interpretation": { "fixed": "126 original runner verdicts and composites are preserved. Two interrupted attempts have null fixed scores and administrative exclusions; one entry was never started. Aggregate 98/125 is descriptive across the explicit environment boundary, not a single controlled baseline.", "judging": "All attempted results receive independent Claude CLI Sonnet 5.5 source reviews. Numeric means use exact declared axes only, separately by rubric. Missing artifacts have no numeric score; outputs without declared axes receive qualitative narrative only. Source-only, at most 20 KiB per artifact and 20 KiB supporting context; no fresh rendering or runtime tests. Codex reviews remain separate. No judge opinion changes a fixed score.", "performance": "126 terminal trials have complete engine-log rates and sampled memory metrics. Interrupted trials have no invented final metrics. Median output speed is a median of per-trial decode rates, not wall-clock task throughput or a token-weighted overall rate. CPU offload occurred in all 128 attempts. Shared GPU memory and paging were not measured.", "paths": "Evidence paths are repository-relative archive references, not download URLs. Raw logs, prompts, artifacts, credentials and binaries remain local. File hashes identify original unsanitized evidence." }, "evidence": { "sourceFiles": { "evals/runs/saluki-native-0.1.57-2026-11-03/model-scorecard.json": "481a3cc47e79fe559694af2e3a62bb945062cd53d1a7c25f7208756450cde70b", "evals/runs/saluki-native-0.1.57-2026-11-03/final-evidence-validation.json": "b4b6968ab9c67f3c47f1891ebf7f36d58997358ae5570b6d5d8111a101aecbdb", "evals/runs/saluki-native-0.1.57-2026-11-03/summary.json": "36fb467a92fba3648ab776c6f897adda6250fb0aaa06888938190986df1bb905", "evals/runs/saluki-native-0.1.57-2026-11-03/campaign.json": "1d4c4505ad0006962741b77c94c9564cf45492d18ebb6bfa719fc30237d677ae", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/summary.json": "e3247f1d014b4b76f6c38f38b1a98cb0ab6bf03469160a7fa181e4707bb857e2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/campaign.json": "12411b6fbe8f5db3b53eabe316a7350ec42d7f4d43b402a9c64bd1000ea3c431", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/manifest.json": "5f97b6c1490170825669af3eacdc694889335c83bbeab7370e9789790cb428fd", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/facts.json": "fb5fdcdd767852a1fc3404dd69ef145c5515819dafcffb976449c91088c81d52", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/score.json": "cd3fecc6020f7424222fde155148ab4c5317542239542bacc15e16080882e313", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/result.json": "95acc4390fb34fb1ee65665d7a48534d3ea2d1b54881ed578c63cdbcdf1a8ead", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/postmortem.md": "ec4a4849bae4fec661268e66ab925d300871eded2a48a7137a67dfbb8e55b327", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/facts.json": "54bb066b6b87bd9d5fc0faa68e0d8bd542803b3a9dd05b9ded909c96b63622dc", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/score.json": "f606e67c4a55ca177d3534ad5999e061f54f6004bc7a8400d781f8283458332d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/result.json": "5b5e7a92e236d77659d46ffc65ab64673796f1bca57778e3424555d5b5fb56bc", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/postmortem.md": "2b648c7bbd7f907eba064b66a614c37c2cc97b44d236a4867d7d00ce45ab617e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/facts.json": "2dda1e0c9cc5c3b6f849f3b76072d684f3d4fe69152a4854e3956f1eec8fd75d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/score.json": "4fd0e61eb0e08d6a00077d3c523d31806f17195e76221a6590b2cbc6e82ad3a5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/result.json": "139a4dec3fc8f2350058bb81a01443bfed8e2a447921cc0d5b9feed6eaf0022c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/postmortem.md": "f1c6acf5a7e20a921458398ede9d76e6898989d2e1161ef59a16ce46de204c1f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/facts.json": "239d6bbda7ee04b87b7b04af998591374bb00467241199c3317bfe410f52d85f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/score.json": "f9e57653d66e982c022afc2a3f4f5831c5c16c7f6486654be0883ccb337a6e81", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/result.json": "5509785254811ecf5fe89423b8fc139db188b7f08ddbedbb89004b092684cf29", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/postmortem.md": "0a674f14b4511c7cc3dcd96c7e2a5c2edf79798e5f2669a4f696c8526a5ca091", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/facts.json": "2bd7301aeaafa51455a98270e0475df8ff873a91a95cc88cc33e52d7e061d00f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/score.json": "6985caed406cdc8ef506c7ba3d01c54a77aaaa625edb3a3fa3f6e4bc8fb41070", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/result.json": "67b00bebc3f5f401b9128ff48a2b1314870eeda6675ae24fb60e5fd0e910211c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/postmortem.md": "6c3e426a31a535e67d516da2bfd81d2086d429597ca28aaa3903aba9963ca327", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/facts.json": "1dbb45033d32a0c3516c0eceabc81dc3c4142bca525c2a1d1e80caed2c4a6cb8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/score.json": "c38409713904eac8f5c9fc2152e6a8df79826f1a96f7ed38d78cae86ba6b401b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/result.json": "14bc19b3905fce7085ee05f54137e93b11eb40e3163b965853fae2087b4e3b55", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/postmortem.md": "a4660dc52a58eb81ebb310c151fc047832bb3be17aa9c99edefa3fde438078e9", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/facts.json": "a85a6b79b6af7b7e4b04e8b16889e899875ba49b8a101545dfc06ebe7664c00b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/score.json": "daf757bb560cb1bd459b15bb383111a4d32a1121dfc4c038e0b93cc168fba17c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/result.json": "52e37f43e8db331ea57ea16fe4ef4d852d4637c7a6d08357d61963c6af8a8199", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/postmortem.md": "01c382d7873ebe53744e0dac1b37d4e5a4d3c53bd99ecfc743aac12aa6987e4e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/facts.json": "9e94050ce147e9c3cddd3a6e0178a7f1ba72beb95b34a7587212297816b73946", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/score.json": "3385bec5b93fd466dc1c428133e7d13de6802b46438fe5c8309653c0f308655b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/result.json": "7112796c8c2f2169c109d0afff4f01b2a5d27b5e7d86b4833bfab1e3728c51d7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/postmortem.md": "d39cc9deebef8902f01431ae159d172d3afbced1925096a734caecb80a645562", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/facts.json": "adfe03f48b67921aebeaeb09fc630a0f8dbc51bba14357a791548d91d4dc0863", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/score.json": "17e3f8de297d3a7989b30b5768e9056b21070332a08338877559ff63f807018b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/result.json": "0c051739f76602f63531bfccfcc1d43da01e5aa4ce6ad6957ddb136b448fc8e0", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/postmortem.md": "53b7603b3ce5e60610642a3d9e57df96f65b57117c5f80103da6a6e3389f2084", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/facts.json": "43939950521d1144acd9dc58fa5d25f8153d57fd09b70e3321f4d71c49f48b03", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/score.json": "257be3ab6b99b8951d45259fcfe586deccc5e2902496a32d1476e35303207dfa", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/result.json": "69d8146e2412f05e8a84bca5ed5f54d3530a5bb4e3b029779f1b692dbf940b0c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/postmortem.md": "c397a871d88d35cc681efd02a0d9af163395f7475362866f3b75b39eafb8a84d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/facts.json": "46fc467fcd9e384009e329b827e3d4bb2d7a69cbd9a7209f983323edc61e4aee", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/score.json": "5d3bbcfd4f3b395cb77410cf6268a0cfd3e1eb31131952b5f9f95b7bf067a186", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/result.json": "8a76cb726d52c04262e83d015a89729be4308a61bb49ed288b812cb6551a4eb6", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/postmortem.md": "7d2f422f84ee371854a994e6c9d0946c484280f7be81a338bcd86147604a5193", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/facts.json": "6320815a6e91badd25fe40d5f9a66cf2101a2c8243d39f8feb41580f6983ebc8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/score.json": "3879b52b92bfc1b57d6cc3a1250623b316588c9ae3e381094d776ee950909251", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/result.json": "cb72bb0fa20591a197a5338739dda5f6b6b36a5c98e8979a9c1ccacaab65101a", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/postmortem.md": "77bee28726bcd4c5a375ef02a62142d4c88ef84a6654c4e8f6d243c4e6c2da94", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/facts.json": "d0f61dfcdf83aba27014bdb13822f83c4ac523ce2ada860cd3ce91dc5b870f8b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/score.json": "6999cc7abb85f6a75f56f2024433eb0e80be49e595e69d9235910731ec1b43f4", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/result.json": "e70f8f086840b68224402b75c10220308ca63e3aa52121ee206c9bfea99a8cf2", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/postmortem.md": "cf583834afaf6315299b4f372aa00e56f2696065b97295e765d2c73606ff4ebc", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/facts.json": "d14630f5bac3ecd19082e17532f0a430b84ab60eb637d4b07a1d7da078c0b7ab", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/score.json": "6f332bfdcd28bd66b80c22ae1c4ea077be5746a9af954a0c04381aaef6bc4521", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/result.json": "151acfbd073ad4bc934ab620391b144f030760d3da20de0ac6f1897d0bbae722", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/postmortem.md": "ceee1c3844390fa2178560c16add31d04033bc3eb5746d0d9bdf3e419fa44c1b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/facts.json": "b4ebdec55aa5c1d86fc90c32dd3f704f9729ff2a876b4edf11049ce54b037fd8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/score.json": "0917942cd643ab940504f6222d740f728223c2f71c00799b336077c646eaa105", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/result.json": "69152ae8354c706f406637a08f511c59ab61ac253565152933214352c4c9ec1b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/postmortem.md": "35ff743a17c9dc6f0a8af9cebd477317b13d3bf01db3ddc606eaab4be0142230", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/facts.json": "dfafa55c34510a6fe3e9df4731324a7bf60dbdf33a04a5983a1b19cd266c0552", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/score.json": "7fdd7a5c5d2e41d9fd3d091cb165a9a88a7a3d5cbf4476614a64a57305f459f8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/result.json": "f24e80266f61ea456095d4e87be4f18020ac1c0ac09593fa172bf6e9fd0b5429", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/postmortem.md": "1beb2bfd9bc8bcefa7e03b4151f0d7921c021fa5395211e892292b2a37198578", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/facts.json": "8635836614662ff5dbf38d9e18879bf06e3893f1a83710d6ed42efa4f63dd223", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/score.json": "c7dc5b3e6163bc6be1d3c263f2310f5bc6d7ccd7b91c792c8e4279ced8f78300", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/result.json": "3adc4fbbe4527c00a959b6f0bceefd48405dfe67c897a8416f2ec1ffb1d0e245", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/postmortem.md": "4c460b445ef04739699177348f0d9c484704f7b3fa1b530910f2e1424d691b95", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/facts.json": "38bcd860821193cb14818320bb4092a1e967070da38117774fd58803d9510243", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/score.json": "816e102002e1d337a9b377b0bb933a0bacf7985338dbc00d88728c9ea9d959c4", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/result.json": "6a0c308638c2f34e23a229af12e11f8f5ade39bd2f8623d4cbd7ab19a6f3f589", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/postmortem.md": "da1ccbabd47f82cbb35a35991c67fabd53b6c54544535b103a0d2e14ce9a18ce", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/facts.json": "32d8a75e223e24cee83dbcd4af7c0ee7e685a68a6da11fcb647f244e9e415119", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/score.json": "e78f657aed47f8ec111e1cc5ef9f15e8ce151cc5504cf96bf4a21f1d1b64a3c8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/result.json": "577ed560274bb9f6a8e7e0a9994c2f73a65fab5aa6fcb653baf05294fa614ab3", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/postmortem.md": "9a713958479146f227960813cbea7b8183dc19f3d1d76bdb8d6cdbe1d791558c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/facts.json": "1bf2c8512924b45252a192ed900045a563daffbbb9b2bd8a0528f73f0f45e44c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/score.json": "b1f0fbaad85f7b4af1153c04663151ddf0817b9d1f43f959c912ccc91da0f2f8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/result.json": "e335cd8f05c4e9e6af22f6209e933b102c2d38dcc710991c165f9f26d5e711e2", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/postmortem.md": "51cc840bae705b94caa3218e3e8a4784212aaa7cbb4f34744fe156209f420c74", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/facts.json": "cfea18669a4ff88bc10315ce876b183fc9d93e3e4949bfc81e0c9247a71edec1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/score.json": "145ccc6cd2059fd0a8784828453da8d29c9230d886235d0361758ad50d9d4256", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/result.json": "865d89f40f738215a07487fcba1e65c9414fe8843c5dcab726fdcfc6047746d8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/postmortem.md": "cf57137877519682fe3db2705cb075807bb65f03fc808b828f419fe542e76a91", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/facts.json": "ab1d30b61968e5ff1d4876971523173c817ae3d058e9241cc59e6bec606b6c7e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/score.json": "f5c846848c1db986cd1ee3fb51d59e5cb041060203abc3234c38432fcc92f7d1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/result.json": "6c6fea1b44973fb46aeacebc3b5bf94eec9959a1d11f20b51df50c123ea829ec", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/postmortem.md": "fd1f4fd087202de9ed5c5fa27543de865be5055c5b4896e8337e173dbe02df86", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/facts.json": "72af3d5eb56b404d168e196fe4ac70d21432114dce73b013ed243906c955a09f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/score.json": "c976d6310ac2104ced9c09da22805cf9afaf3625441215284b42046d570bc497", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/result.json": "423ddbf911f39b6b7f69be7dc2a28a9553ea4736a4556b7c40e660222ae62477", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/postmortem.md": "408cdcbe382d63a889edb2a50be3724e111326b15f4a86d097c12cb436c68ffe", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/facts.json": "e0f1cd288edad23d5cf818d3248be8618db8b08646c5c44cf9660af2828c87f5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/score.json": "2e5703b184da7bb3c3f94afd494b79abd43d298b0afeb64d904ab5ab2b98bfe7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/result.json": "f17829c77e692f7f6d2e5ae2e15c84de7d38e759d09537cb97aa0004224766a8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/postmortem.md": "63ff68e5f7a10d60a87bc6250ad91912cfd38b59cbaebb59bc1bae74bc255d98", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/facts.json": "1e7762124cd033558fa5073b51d1bf96e9d844223cf09baea21adfd80c0bea33", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/score.json": "761e2e7c4d0264438adb9eeaf8e73cbf8bd43ed7f784c3124a9ba429b83c015d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/result.json": "8b7cfaa12497d63442df867a53d63d271b9a71e13b26f6e77d7b4db15e7a5877", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/postmortem.md": "cce6a4193a7c092425bd1e4c102fe203a264150241018ab9063bef82a6d1735b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/facts.json": "e5c7627390fbf96b40b426d63027d4addf41fb99ef16410bc1afcbb453ef2e1e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/score.json": "435dba91f592c2e32e5a4039c7aaf2319529db5c9697734720d19f2aa8586969", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/result.json": "638190c92dcb6407b4941ff02195613706247e82bf55e9a48a2957b4e9f92bcb", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/postmortem.md": "c5a1390ffc4b84b2d9dac2527aab3fada274699a4bdb397cd9a6b50409819360", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/facts.json": "b4e9e944e20d761b3b11da70e157f6402fa9e4be0b0109385d283eab945347c1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/score.json": "f7b143565890099c50feea111e99889fcabfcede21013b213e991756636e638a", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/result.json": "ed549279e148fee083ef2a048dc83f5fa3158389b3ea0d396721aef382d6dfb3", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/postmortem.md": "000abaa496e2110ab154f192ee698a12df9161face191611610055a619b1bf50", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/facts.json": "1c937e68f5dd815691da0b84b3afa2f4d7d6c2872445b0e61dc357ec4545d8a6", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/score.json": "61afa9f33770908cfadbdceae5b7b66412fd386493603f04b471edceeae52961", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/result.json": "6bafa5eda0caf55103260ce7775a974159356560ecfb38fa4bd35e34949adf6c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/postmortem.md": "c07c830ceb6650fc511edff737a878870960b01f58db4ef62260c67cae0ba875", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/facts.json": "edbfc1fe6c012c7b0a5297de251285367469f2c99ae8ecdb313c3837662102dd", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/score.json": "da35a1b25b9fb297d8d1d417a50bf5a8e31817c9e027c3db73d7ed1905608c84", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/result.json": "f3a5d78100c513d0f17d55647ce1f60c4b7b70ab920ec7185caaf55bda3bc0d6", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/postmortem.md": "02209d50b2f4210467cd4335660dbcf9c032771207e761260bd6b84e5d6973f3", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/facts.json": "426b9ac68492fdf0fe29372f98be4ec1463966fb17f6cd64442110f5c4744f06", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/score.json": "0e804ae0f41ae5c0f5e23f4e522dcf6f62c9f5085ee554657078bc2d58f0c9b8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/result.json": "a8fbe300bfaacb0f34e99c3a3537d60daec1753021bcfe2462b18e9edffcf1e5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/postmortem.md": "bb61e5fbd7d6b5ca6eb849e814ce04e9fe63fb0cdef51f592f973cc521b56591", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/facts.json": "2be670b04aceb03371b126e85c3375d38aba4ce78aa03f3292a1b7260c204834", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/score.json": "9dc4ba4e20bf91df6d0ac3dd574880dc6e85c1c00d20a2162815df4e6fd427cb", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/result.json": "1c7ba77a4da4134f151e06233c951f682fca57f48648d9bc32c2addc9474d771", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/postmortem.md": "5054537003c9c3b171ea024e4b0e8a9ca8f0a28133d6100a52b755cc9a1e6007", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/facts.json": "eb72784cd74a7aae3ac61be1d61035c2cf86da7093e257ffd8606de679c4040b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/score.json": "8e6219b41c95c37b9876bf5e89fdff5587999b18221c49cdc4a6d3c86e3416e1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/result.json": "bd22d1950d10a42f6ca0a0eda64d641171ec94c654b0d425efc5f0f661d10b6e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/postmortem.md": "0ac235147cf33e59f294a2733e9dffb59d6e4694a4f57874ae1e21b8a24ba34a", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/facts.json": "448280c95a1a1906e351870f8c82d31ba648663be39c7754067b1e28200575b0", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/score.json": "9e4da82f8942dec5f92435acc236e8b3f1b5a3ce4d8dac525a64c6d68694bd21", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/result.json": "f48a181180398ad6d8c511bcbc6ec0f940a1c5a9e9dda7b3237065468d2eaed4", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/postmortem.md": "8aa979cb3fc0e6442e8c0ddd832477cb5fefeea82bdb784c7537be6e375954b4", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/facts.json": "58a21da025ec1671e3a3ca8e8e02a9700571e0c3d908c76dded00a16c9a0f6a2", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/score.json": "bc4c71f5d3c64b7c892552bb10d3bd34085941a4fcce170cfd55c3f76564c5a2", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/result.json": "19577b0b524af9a0dfcf555106df495a76692b7904612f92d32e24043c95301f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/postmortem.md": "ac6800115cc6cd5e4d985d0bb12a2f8cdfd339e4b34e3c1b9b25f613ea6cebc0", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/facts.json": "89cd3d7fed4def243b778a753084b64df858593a704f066491143fffb07c42f5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/score.json": "641f46b1b72b8670cf07b92235379c8c46fd8b99d0f7581ac8f78ed1c981b59c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/result.json": "7fe18577b222227b15ca44d2e91fe737f67b6c406805baeb14f4e13a5e5b8569", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/postmortem.md": "ceabe30d36a2ef02f0bfd0878cf20318fd9e6d2ce2567eccbc82634f21d00b7c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/facts.json": "a8aa7aa82c1d713d1a6b3c7b5dcd4ff63cc87fdf47186c17a0e0234a3dc0bbc4", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/score.json": "12dcaf7ee22915ae0838d6b7b18712fd741f5d758f58e4910830806aa425cfb1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/result.json": "55486f38bda9d3dae48c41e088ad509f57bc6eec4d8f975d486072bd48af073d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/postmortem.md": "da6265c2c41e8892209f4b5fc122a09d5fd9f423ea67855fb949065881e5db1f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/facts.json": "64d135e3c558bd27d93d658901c24dc49bfcd98641bcb00676336aa8712671f7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/score.json": "6dadfad57e0a66163e0d0505557c040b5776ae2a9212f714ff23b73ee95512ad", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/result.json": "00398128e64f3f5e0d81c743f67a097785245f8893a2df4cde361233b7d25315", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/postmortem.md": "dddc45dfac81156f8eb2124e73bad072714dc8ad6239c37fb1f2db6d5702871d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/facts.json": "f555bb028edbfdb220f105822416419ca9cf1c0714ed35aeeba955016d8b6c5b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/score.json": "02e185718f664914d02502e94fa7eabd9de50b401ec4fd536e7ab9158121e474", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/result.json": "8b8c39bb0114c2a3a6e33be66c9444c15c1f7d6f66cd26d8e3fecce14d2e95bb", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/postmortem.md": "abc9089ec6d744cb70b9cda7b60236cdc35691a036b1b3d5094a1603a7755b44", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/facts.json": "3914df243f23c9a2192d895b353414b0f2bb42d23957365927939203dc0804d9", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/score.json": "771c4c474935d386ff2275aa145dab40d8dd37c2beb77c2431e9df9203302c1b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/result.json": "d345a18f13b3a10329aa32d97a1d20a74a16864e07e4b55d90673b3309da3c33", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/postmortem.md": "415691084623c68a8afefba4771e0baf560f952599e2a35c91721085dbc1ae62", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/facts.json": "d0f1b3aa1116606885d9ff1af70e7d008f1a3034eb873dac86148411956c65f7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/score.json": "8be0026432ffde399bdd44850394e41b4ed7437c98e4854ac7137978b70701a1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/result.json": "eed08b89c02d31a114fd688ba15dd3a53ea49cb2f9c9b0534b724ed8fd5f83d1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/postmortem.md": "2e5269da937a52938394f58fa273db4c9e0c28682126b3cd5d681f1fd8664d84", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/facts.json": "b3a6aaeac1322c6a6caca9a24e4e13211929c8fc0fb2237f6ad31747531d769b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/score.json": "b8aeca683024c9768a4fa55b754a65ffeef384076f3720771a6667a47fcacd5d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/result.json": "fac41e4b941dfc81c082249998680b677bf4ce28ffba296247cbd8de26dbf7e6", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/postmortem.md": "b98857ee8472d58e0bcf2d213a9e752009c25341573b75edd1d2e7516d6fb847", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/facts.json": "7f164ea43b7e9bd2d9c8f22b6e7925f4408dac0a84798899e471cb4388de69a8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/score.json": "47223f1e545c601f25c82d2e2059a2a5c7b22d856cfa347afb7f1744429b098b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/result.json": "c0f132d7beada42fd55e9039a8adcf6f130743370def745e212396c3e4c01856", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/postmortem.md": "540b832ea5cf98163809be3c6f18ce8b0b8f3f2744b9e54bb72ef20f4484a18d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/facts.json": "fecc86afd05f4676e12b7cfbcfa1c18dedb47597c8101056df685357bad2c672", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/score.json": "d107bf4bcbbfd7eb04ecf7150d439e54b7c5003e5edff75aa7ae05305eeaa9ba", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/result.json": "81940d50bbf4ef0f48bfef2364bc21bc7bdaca2a9dacb95d89cab80276ec22dc", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/postmortem.md": "8a9e6515cadde331bcdd3d0f5b8721381ac73173d0aca720c5364aea8773ff23", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/facts.json": "86074e436e055d1ab986792b61f4090ccf0a3f6b9f8fd0a6fbb9d3502144258e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/score.json": "8a2bf77788ac0a505ac55ef064df9bd9059058b878fe64365fd0fb6ca8948ba2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/result.json": "3c6a70fa49982592d11b588aacc5e4b9389369aa3b6aa291b4aa31f57ced6c97", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/postmortem.md": "422122d108946113a53034834467f458dd4d02d4e1de6ba10931fd17c0cdb600", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/facts.json": "7ceb90e55517a9ae518bb52b608f123d76ea464ef80726de2f43d5cfad110413", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/score.json": "b070ae4265944db1e50bbe175163b5d17f2c5b06e476935a14b34676391144a6", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/result.json": "5be751c62b445e1d8dd1355f5623aba9d121bce833aacd380010d8f85b241156", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/postmortem.md": "0c6ff3aa23cd0eab8e5cfed6d2a6954504e0ecdc77e4928dbb1b80ee936ccabe", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/facts.json": "e2e19d12e2e704a3a8b1c2161b5c7f7daf612673fc248556dfa5c8d2f92592bc", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/score.json": "df2ef8dee1fa7987e8b0cc98cc1526ca94915584c25b195ee672bda313c4317a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/result.json": "5e43c5f6f8a879767d45f6a0eabf22849be247dfc4a343916674780ac7efd4eb", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/postmortem.md": "0bbb350900601ab461c7fd802b08f3c77d757234b8633f1f4b3fbf4732311d88", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/facts.json": "606cd3ecd79cafc9422ece9549ba1c80873ccf5f68a61ca983938db433ff50d2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/score.json": "7487e6cbfb6c6f7852169189406782a25ab07918a57290f94d2f2a1ec0bb29c3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/result.json": "9a4424ec6b9c608437fff9eeb2ea132864ec0d980ac97dc0f09b427bbba685ab", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/postmortem.md": "5197f2207084f22f94fc1e47286dbeea15dfee1a962d6f0372815d1045a5a6b1", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/facts.json": "ebe88cd0515554c56a2bc653c957bcd43b2948febf601ac77f627cac3db86f2a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/score.json": "eb199b982140818c476a421a561b84ada0b2aeaf10fc71257c6fea12b11908ce", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/result.json": "fb9f030a8477462bc90a0635b3d04ec33be29f97e2b462f3fd2aa8af526a1c69", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/postmortem.md": "f8031d349c75c32c6a0ca97d963d8b31f0bc5e8612f8f110c472cb4de7fa3d1b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/facts.json": "70ff6e6fbbf042eb1a6da6cbc631921e7ff1e659ac838e59e9047f079c9eb7cb", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/score.json": "06b42a22839ad3bc967a857b1b0121e2895f8dc24f239d3b479e33c24bfa2a40", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/result.json": "1fe1276a444c63b6114370d0b85437d82b1dabc133b416ee1d009f5f4781a93b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/postmortem.md": "76da524e6c1521978eeeff4050db580bb328286a25be15fa51d6d862ac96b373", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/facts.json": "e57f067563e0846931595470db9a85e3f5700db6c0d0efd6091649adae2d8dcc", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/score.json": "91370750a2a9362649f21523be2dd5fa0ce00d3f5688560931c9689181a974e5", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/result.json": "cac2f4b22c218f7a84510034d3a524923417d0d9403db705cb4511cad622bd99", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/postmortem.md": "3e9f12a553cc39f24698428672c813daab0738d7b5e70e41371d470c3f38ffe3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/facts.json": "d56ce7a5979f39ebe95d029ea3b59a01bf31179a8eb9cb681f286be4ec498545", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/score.json": "061684bb897ffd9b5e7c37cc9dbbfa41fb5065bdc768a58bbc05cb7a1430dc6b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/result.json": "278cd4e917b38803ceaed6d8df68b96590b5c8f11f10e98f3640fde7651a8cd6", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/postmortem.md": "3919f87c15591d5eddad5cedc6d2409bf0cab9566914c092a890c84df20f27e6", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/facts.json": "4d4ab2f721488c26edaa268ac27e6a52b13a420e6bfb3fe3f5866455c965c457", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/score.json": "81ff4d8ccc25e305fd3aeb10424f19b075ad991e18ec4c3332727d8135d9faaf", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/result.json": "547e70d04768f0f79fd2f127ddbb41e861c75c5c97f9a42b931492152dfd0385", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/postmortem.md": "9d121a80b65419800a657f672d0f5c23d2ae0e91129338a96c30df3185e0d203", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/facts.json": "0cf7a968166f2bcec8a0f7212d9b8326b20c20d5bd04dbb79967d11df8cfc4ac", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/score.json": "3604321f6fa5bdf0aeb8ef466ddf3a1bf94da03805297bb850fa34d8bf64ae07", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/result.json": "1b3b51af3064d3f1d61a714dd96af0a038194b6f2eabe51f3fe71790fd77200a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/postmortem.md": "c5a70cfb5546e1527961b02898b674135ec0f71c05b37fb58bd026205c1e70d2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/facts.json": "b491de1482aab38e7d2e96c7e85d3cf695dec587ec1de80d4a229de32af2c381", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/score.json": "43ad875c67228e530f7690d186ebdfaa3453ea366c8d92a092e8c0911def3adf", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/result.json": "d93b6383429e5d26d29b2768d8f034c067f61676bb13dc53710d1f9bcf0260ef", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/postmortem.md": "eb2ac40a6fc2ca84b9dd89791d25d226cf1a9eeadacebc6577424789d88c336d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/facts.json": "04824f6bc345eb1c612e33f2a5e076c0fa2f629be2091ea518f7773a67f1b69b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/score.json": "26c421e62e2a3997e38b981b166a0177ee0e9da91d96f364fea99753428385e5", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/result.json": "458dccd3c62c8912bc90659ac5c7805f335f150293cc484a78ce473714a2e608", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/postmortem.md": "0098241f6be0cef1584dc000d5edaed07a349b13ab3a0fdfd8fa6e060c0d1066", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/facts.json": "89470fea1aeeebf02d90bcf5dcd7223ceca9b4627bbefa66908f6e273527d66b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/score.json": "7be2c042d4353c2e53a9f6873eb0da0b5eac5dddd2341f72acd85844bdae6546", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/result.json": "44a656e43455942f762afec78a61c556aa3edbf3d1b823e0026ee97426a0b40a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/postmortem.md": "67b3d4579a79aa98ce796661e9fe655a3ffbac0151ead9a6869b73ba35999d20", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/facts.json": "ff5d59995806d8dcf38fa757c7267ba17bf7587ad8c5533a43c1c1d5e6ab37b6", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/score.json": "c9572aec78c5a157831f3dddecb97f783a2edec526e7973cfd05a43032e20192", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/result.json": "59cf2a1dcdfae3d485bde133c4f53853799bdb38ad30bf4526b23a7a2f097766", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/postmortem.md": "d870bfddc3fd5ddd775fb5a1c5234f35c1355863f58afce84cdd36041f341807", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/facts.json": "5548cc2868b28a4b83edc2e4c4fc1bd1c9bac7af26d9cdaeb1a62caa3d0726b3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/score.json": "149e451822195f17181758f04e5235b3b9d7e1c8b4e961e880f413e2dde4beeb", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/result.json": "a548a92f9942b00a3e0bed412bd155bdb235852c2eaa98df043bd72677a5c52c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/postmortem.md": "7910b3301fe7eea989915ffb76ca0313d5ea4996c935e9360edc730f2f29698a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/facts.json": "d877d5323800a7d7289ca9474fdb09c59e0781e3ff592c14e50425d0ca93992d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/score.json": "7143ad4b8bce775523abc2358b919846a3baae204eee4ebd65027b5c244d2b3c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/result.json": "ba8e476dabf770c4dfddbd05d6fd50fb760ea9e4d1b7172c21ec20f1afef921d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/postmortem.md": "e439d0ae1a63b20b822de9522dc339c6524304271f870a2929492d0e037daef0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/facts.json": "228092cfc9140d7a721b1aaa641a5fa0922c6fc30677fdec95f936e20ba650d0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/score.json": "7960f6faf3a49c40a23f323debfce045f6f7285c883f07379b15b761212a43e2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/result.json": "715aa3ead027bbc48001f8debce23e702784cf65426ea2760f9e4000400c5aa7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/postmortem.md": "13dfa855f6c0d279d5b4f64101dff7201631f508847a99813c07c7f3ae5114f9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/facts.json": "55a31c077a3ce9a0b70968b6dbc0426b0c6dfb0cb3e10e86c4d939a4385cbb7f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/score.json": "c04fdf4042035771b53c640ef88967c2d7aa28cadce8fa8d56c4c222fb07bf47", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/result.json": "358ce44f9b905d2747af77d7c0aca1c9ac6c5be598fd6b4f2b33b4f9d33f14d4", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/postmortem.md": "c79e6894036cc022789e73bc114af2a78fc27bb82c20b0e26194d652079285ff", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/facts.json": "0ca9c483cdd166e716ae25ade3e26422cd39386561cf05ef3b0f0337c7444e31", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/score.json": "cbf45505f84cd0da9421ab80bb98dcb246223127e64d18797b94b003f7ec8206", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/result.json": "647fd6a0bd9b239539a31d03fe3cadb70d3e3b205990af72236dfe17a14d5eff", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/postmortem.md": "1671eeed603d852fe9b32aa5f19366e7f9e2350a0a806acffef579be3a7b40a4", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/facts.json": "10d076598468880faee3acf3ad9b11439f8ccbc41cc905aba6c52f6d9f362037", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/score.json": "ad62c5514792c84ae9a6dc7215e7788ee04a897d61ca4d22e66e99898b4e7718", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/result.json": "6a168c4b16882cda13a5aa92b9bfae750412a7f273c3ad2986e3b67bd85cf95c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/postmortem.md": "99902a44c23eeb0d09f43e6251916ba1d3b7c8d3c908bad705454b3c2b1948bb", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/facts.json": "9bb2461c1ba2d2c5482d5a543dbaed17672a0356853091e78d409b82d80cb780", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/score.json": "51ef218530d45b7d3b5d7c7becc3abb483ff91566a6e1b6430047c6b2a21562e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/result.json": "eca7af98c212a4fe9aa2702d64c0c57b488cd09576903e69bd9fa371fbb4d897", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/postmortem.md": "16cdecbcc48894e3f661cea25603f937e4d368d3264edd57aa44f180cd421638", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/facts.json": "7f08dc194e6b366bcadbf93ab5aea7a1689258f3817d9c8358efe3cce0b8c401", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/score.json": "f2d68323790a6f38c192c41b4d5060f9b3a4c9cdc8bca3873e639def58b075a1", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/result.json": "2afadbbd1ce3c6883f396e4c07abe15cb8099bcb15add8caf072f940fb7cbb1c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/postmortem.md": "6fa5ca90432e6757c47095efd239098a14d2ee166faea833a0a899da2d4ba809", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/facts.json": "2b5cdc682c4a00add7da594dec495ae4409dbaa11d2d49a3467a363b9e0971f6", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/score.json": "92e60efdaee73c1508e0801f9497d7f7149ebca87165442685abb4fa6e5cfb2b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/result.json": "6d6b604c3e594c453dcc09eb15bdd2c0770aef9f269c793c4fd96e2a8c83cf33", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/postmortem.md": "7ff7682227e1444d859a3629d447263698202151e269523543b42a8b33e286ba", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/facts.json": "02e379c9d62efc9bbbec8fd5c4e8e35ce2f493e4e2e4a303301d2da7933e2c1b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/score.json": "cb3315ce3fe3d1330ce3ef297dfb540e899e77b04cc351331647a06f85073212", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/result.json": "ea39533cf207f1bb82dc07ea1ae06345417ec4dc27715ecfac2f2196e0702877", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/postmortem.md": "665ef2132a4d736c4c76ebee7c78d4e898fb82b4784c7783ea2c8321741689c3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/facts.json": "70b80619cb1ef77404e56838b17746f53cf52dc63dbf11d2394532c3197db825", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/score.json": "a4d9b2d059a1f525476cc8c66a5056791ae5a89a005ecaa91d854e486518dd73", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/result.json": "e112060895aa35ce16f1edbbac5f550405396643d0297f5848da8267e644a373", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/postmortem.md": "c4c35fc7b8a8a7b52a8b143b3a8e28581a251bc79a3d97234890bfcf9f8d1aff", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/facts.json": "b173941cc2e0dc80b1b83238b35cfcfbe9355223cd03067e42277336dccce8c3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/score.json": "926ccfa5e28df2624325fe6d6fda1e33217cadf77ad3ccf46a71bb4de419ce33", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/result.json": "fdf76a83f4197a361cac9b60410662d2e06dccdda7b235decf86de6acb6e86c0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/postmortem.md": "5ac577277c5cd2c42755dabd7857435450a52eeaeaef222e005d01cfef2e9b81", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/facts.json": "a822ddec2b8defe28aa4d65f168ce8b45f5e031f9438cc62c1528f9a51807f83", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/score.json": "e4b63aa62f61a2301bf1356086234e7a6fdab6ba2c4a0063fd87605f2568192c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/result.json": "bb846950b332adc5363e1f4f83c22a4a8a8d1bc5d69b3291ec461ee62a6609f9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/postmortem.md": "9dd9443c7a960f6fb81a7fcc297a33b847b390bc7c0abc2b079108e25e917212", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/facts.json": "e90d250fee4918d13d14a5cf1e8a5c3bea2cd396f23d8c1d15c799742e1713cd", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/score.json": "c64633661497fce9498206465925dde0d9f6243055b9dd2b8617dda7087df7c8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/result.json": "b39ce78b6b394a5bce91db4ace304ff751d2afe0714ef6ec13d29d49871e4c55", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/postmortem.md": "44e767c9db55bbf1e015527be27a76c94ccdf27878d31b80099dea1c0effb2c7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/facts.json": "42bdaf40e54e8f2279946c72560389940a2306b2bf559d077559d40d5eda07b3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/score.json": "91b88a22416b0a8f8e63e692581e360bd80450824efaad4f4c76ff8a8b92ee4a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/result.json": "503ebc9a8620fef23dd37fc94076d9a9658cd4570370076bebce472604117f3f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/postmortem.md": "2e023185fa43276df28bfb12a5882e02af73809b8ae038c74343386b112cfd02", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/facts.json": "4d82e8b869c951b74030b0817016c22ef48db4ea3dd382a4bf586f65aab5e853", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/score.json": "2226b3cbbbae941fee08b9d4aec6a95faec79dd428d2ec971c8937f77968a3ed", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/result.json": "60f7647a0ef01420129884abf8c1e2419060f392aeb2ba1191b5e65098ea81be", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/postmortem.md": "3db00c31658d4909a16fbf4c1da5e63438b148317be16b747f6ef67b293fd088", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/facts.json": "2c97152c4153b265ec0fa47a9275777df59b4ba4354662859c116f0e9b182441", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/score.json": "2400aef7c8be3bff382427e372a3cfecfe7ea2872eebea7a55fc9c13127fc23c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/result.json": "7fe80e75d12af591206446ae6a880cd83a12859d18381432a033cf53aa2689d3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/postmortem.md": "f782ba645af2264480298ea42536bb9a3777a4e7ffb9354a27382180a736e684", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/facts.json": "e40a069ccd21e17c31c706c0767120eb72dece614dbaaa842c7cd3d2dadc1b20", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/score.json": "82ac8f552c009f50e72381d144f924eac535aeb1b07ec62f78f7cc4c09595ad1", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/result.json": "c891ead28ae16d59fda75e38aef09bdc1ffde4e4de9b26f1d307121554723191", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/postmortem.md": "25c61e2533c7023e32ff629eafecf46ed992f8a9384e14d9c777e7980de5f052", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/facts.json": "7336e77950a7c1f1f15e39ccf7babc5578a2c1aeb8a8e99c3ddbad04ef8b3ff3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/score.json": "4c4f3e788a03edd1e676cf53057df22bd1d04fba7e802a82924d4480f38f4c24", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/result.json": "ddb1fe16896dc8da341526b9c0b813ad420ef3ac2e40b0778b6a2ca943692ccf", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/postmortem.md": "653360df727335327d1ae777ace1318d216411dc03bfd63f0fc9e88e9d1bcf21", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/facts.json": "2a290c6acc0b5acca8d6ce39299bcc8be17750a5a1f477204b96515d14ec13d9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/score.json": "c8089f3ca208eddfed219973c29a17aa430520016062c29917036f69e65bfbcd", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/result.json": "bf7f8ffefd3413076482ee0aeaa141e6fa1f53fecbfdd21cfd177e8f3bcfef12", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/postmortem.md": "b5baa85e0dfbdcc74075cec8d3208c06d5c9d9b7ca065782fd234733014c3581", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/facts.json": "1286c9f2a41cccef5d11f30a48d55f7a979011ae311cde51db0391c0ffdbfaec", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/score.json": "4a673124799d04280201eeba0c8c7d28eec73cd73650910815183da129203966", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/result.json": "1d2705102028cd51ca54ca2ebe009fb587068b557dbe39a4ae731c5f7cf85c59", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/postmortem.md": "898c1423b5f0e635de054653436a5e0f5cce16f1f15560cb6af9cd5b4b991100", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/facts.json": "9e78b8a7cb3b3c6f5c6fa22e3692c247f5a76d91fa0dbd35e09ebb65204b4495", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/score.json": "f43fd42a27ac843aadd0d47adc21783849d806a13465d4d507f5db250b1c000d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/result.json": "4630944660c9f927b4dded744ab24a103ade6d68962b6a62e372ba9408160d2c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/postmortem.md": "5abc4f317b1be96242d0f934dda98f818110801372f46159f7da269fea187ce1", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/facts.json": "9aa592c8b7323971c0ea20f0498ebf9815ec56ca26702315cdaa649d02cb48f6", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/score.json": "6277f20171a8c4f54a997dea5992031ab8d04804a4a156e1f959aee9a28ab9c2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/result.json": "c5ce55ebbe6cb2e96e204f040157e55207d07eb1a5d82dba6250d6ebb427c9f0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/postmortem.md": "e9519430633282584bf517a6d994f8868dd7af859bbd527913cc1390fa394f0a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/facts.json": "dc3cf7a411023eed824d4c87bd3d54d6a7a05d8e0e6896a6996b571d027f99f0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/score.json": "3deac34ede9ec1511888ec49340ecda0c42e4e24e517b7e46723b59eb0c31b39", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/result.json": "3bfd5e400517a3dd444503c102f5a7423e4dc91b999daf1e40790cf5b930eb6f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/postmortem.md": "47af3078b06d84583ddee5dcef8cf7769ef6d95d64a67d684fb2200ba4f19313", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/facts.json": "3ff07aa9d5422df60cc90f3aa68c7bb560fda7e8edb96d609e11f332f2081d87", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/score.json": "8a26a85375d4378043b596de11f150e7e2b1ee8f139b94c34bf262d7cc37ec0b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/result.json": "59160c147bfeb8a682adf9782ef9b9c4b806852588d63d89f015dd34fdfee926", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/postmortem.md": "2b7fdacb5cdabcd3619cefb586e3f3a110ae9385200a4902137515a0cfd16d06", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/facts.json": "d5fd14a0d97817cd388d0c5a7df46b28290a493cd830ede33cb7b73a1eb82bf5", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/score.json": "eb130034c1abc7d4e51add534158cd368454398a4150516f884e312f63e10699", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/result.json": "e6fd347e686f04fa32d88fad9337aa4878ee6f2fd43ecb769ea437b19f7be24b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/postmortem.md": "c0cd3a85e96639b0aa3b5b219e9f623d4b96ec772912a6ebe5b04672c8ff57d5", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/facts.json": "fd4feecfb723b534deec3906edcb7573482c41e916eb50099969b43bd59c4dd3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/score.json": "1a3c0a3ba5a7c80b484d5197c7b678a3ad480f6aad8f11ab4c79c65762a73cf4", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/result.json": "7b3be03705d5df66a7ed3e38199de61cb46e32c99d7f98036382080113a0e14d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/postmortem.md": "02f4771c19640cb7049b77cd6a623de5a2d932deeeab7fbc25c38bccc8caa381", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/facts.json": "2a4dcd9931ec81c420f94a01668527924df3fadddfedc8422910f43f33b2d2c2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/score.json": "b74d0b67d18bd2419f64604171817fac9aea6cd76862436d745f4921406c9090", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/result.json": "8f12bf167b0ad933c9005791d5d55afc8a81bbd9d28a6ba9c59c9abe1bc16f08", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/postmortem.md": "450a593c88a596309204c611ab8824f47ec526c694d4f0a654277510555b389e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/facts.json": "88d2555670073a517f3987fe3b1983168aaf43f26db81fad67b92e94825b33d8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/score.json": "d6934c0a093e9fe54e21e69ef1570536f3a3a54e17094317e4f260796d1ff1df", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/result.json": "e0155da2395b613b3c3c2ad4ea132ee5730421121b040bc5e865dea75bf72479", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/postmortem.md": "8c5dff0f6d20498e27be4e8d079f27fa21be8b0e97d4124c4bf48eeeb8cb4cd0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/facts.json": "486ae4614289e2a50c0721501c371ccf4fb47577a697a697562a1cd6e0697c1b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/score.json": "2f9add9397892e09cc650cbc32765e12d97773e3b8a8368c11521c6eed1d9287", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/result.json": "6cdeee4e75965eb193f260e82b6856169e55bdeb875b1ab7c14399c67dc55541", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/postmortem.md": "97b74c1b8ae78a8c1d6bfd6ec5a45fe70add480b5e666054c8ff7df3da7aa1a4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/facts.json": "9036c5bc64f766e5968d62aa469e9e457880227227bfe22a29ed0ec43466ad38", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/score.json": "e774436219bb1ff714d77fd40fb7481d75ade26f930b25da065211d9722c0aa4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/result.json": "3a3dc39e373ae13e1334b7a609f9721d04563c47488f0c71d90dac516addd1aa", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/postmortem.md": "43ac1db061b1672f38ba3e3569d0c2bd213f0f1a20846447468a2d2d2736b72d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/facts.json": "03d44ccb8f297d6de1ee6357a92c6e608f6431ff750d90bfbb9b2c220d66bc97", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/score.json": "d106a21cb787f46ddae982ad000d3ec96f98734325bbe52235225bb9bae03c59", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/result.json": "ecff20d56f81eddec6869cef0d9b0b93167ee8327a8c66eee4926a9e628227a7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/postmortem.md": "6a391dd41f7eeb01b5fea2c2c8b1d19a3aa27ed5621f1202d3d7bb7485b8c474", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/facts.json": "eb77637cff80612eb2428747007bf67b89f8d8cf8fada33e88776ad178c3d09f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/score.json": "85a041ab749813d6607582550088cce7d755ebcec660adbd539a8d85d0e2df67", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/result.json": "1e2b87e674e499809424b41656334367dc029b5c22047c43a97e0da618a168a1", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/postmortem.md": "cc5c21c57688c9db18bfb764cf314f8df5d6b1c76aa23dbbdf138b2a7be0a771", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/facts.json": "8868ce5c0b2f63405443a4ad738435f4e270fc670f2fe13ec5281df07d00ead1", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/score.json": "37769ffe218790ffabc543d77e53a26bdce1be13fbec151b0bf5b996f7d99bb8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/result.json": "af909df434431d15d1979978a55bfee7ece56aa0ec04935bd197040ff62f582e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/postmortem.md": "1711c819baa9532f40ddfa97ae4870d2c7d2813876a5d27b2a8784565a225dee", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/facts.json": "166a15e194e98d5574c302eac93db27d3be0da2b121a14e2fb5bb5d4b9556645", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/score.json": "76ae1bf02b5531c7c928843a32068d1b3689b8165fd2723c230db7fb2693b7e9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/result.json": "60a86569817301fbc27f60b1effce78fc7e15204a010f2d693b2bffb1bd8c44a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/postmortem.md": "c27d632301635b8a6514c20a2d0a746249de751acec87f9f82e32512347b7f6e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/facts.json": "95e68a767fb15e0422af67c936d54c52e991fe5f3180e839c884d6130a3397fc", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/score.json": "64aed3202c9797e079cd1ee9929260832284d0c7e8a9d47d90579de40b7425fc", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/result.json": "863ae4fcde3bcdf7709169c729b8e6380154dfd46fc7235505c08babe20aec32", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/postmortem.md": "272392645e11f7b6b748371e3560785dc3f3545f6d943467361f002297333f8c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/facts.json": "aac293ce823ba80a3b163d1ac58b61f2c9175cb16c8e941464c72783d4d5f3f1", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/score.json": "6844f967318e6dfe60b7fd620670c5cdf886349f3b3fcdee08f2212e65896353", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/result.json": "aabc14527330d31ba786a7db1a3f1b7fd290427a1973dd315c67b3b8cd1dc70f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/postmortem.md": "a3ea4a39ef9a545a4377d1d3c331cc63ac9726331199a91bccccd19e2b44b277", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/facts.json": "55ef6389dbcd35c45a0f30f524355d41f31e5f0f4c33f163395a5dab947c54b4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/score.json": "d63f76d2784c2c4ff26a35fbc27176a2052f910fd384783412204464523564bd", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/result.json": "c7d00be0c9e9b7474b992257b4e4b8d5264c444f3cb202fa3614b54ead0f03bb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/postmortem.md": "52733fda5405cc1832263bffe237ae849fc17a2dccfb383f6f7e7830b6f1a822", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/facts.json": "0310e3bdece0c5b0a5014966d608b1fae713394d956898013a48b05e6ec99699", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/score.json": "2ec934871be4bdfc53653b31bf79265acc722a1e2374b04e7bc29b628e191b9b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/result.json": "2cc015123d559a2c36976218e6027e7590067ec63780f9004bdb28e92f62f62b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/postmortem.md": "94867d0350226a5981826a5c470702a39fe63fabf7c9eb5603602b105fa90dcc", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/facts.json": "e73e8c8805d8853834d966347df3d6e5d9afb6fdc0bd0fee4a1d8d38f4fea8e7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/score.json": "ffe3ad7d24c2c1d7085a0b08ed7eaeb98b3d0dbab254350a427e167009fbb420", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/result.json": "d1c8768f43c4afcd7df55ddbc9a13c9017b66480f8a9a78046e62a69c07c1ae8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/postmortem.md": "9dd5002eb932ccccd151ad635786d702c4f462cca6be924d4433ff97e89d393f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/facts.json": "6300d9ab28eeb7f3e9eeb83bf813095f8e92b9e750e53461aea73f33e75fa35c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/score.json": "b2e1893f6c3753ec38c52144443c5a9e3a102a3da25dde4bcb06546c54963c73", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/result.json": "644f6585734bed08add0d75961631eb36718267fc13bf007d10bf75123ccc1b5", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/postmortem.md": "322b169e777dc2b871e11ade3a677258e2866eacfd83c481c100dfd0bf41eec9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/facts.json": "107e953c83a6007206e2b539542478849f14ae78738757cf9eb4bdef3adc1218", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/score.json": "35afb1a3f2e54b3b6b88524a609c0c9291d801e9ed510ca8e1404c061689d1d4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/result.json": "d98c89b5052fdcfe4424a08265ebda15ff9b634e5fa9d152b994096e59a4cd04", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/postmortem.md": "3895fc748a0b1062361764d379f87427a0ed12d74ffabe6ab538d4e38d705e2a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/facts.json": "9199f4ba0af7cb9bcf6eb8e57803770ee39683ac653630b4db4449ed0f4aa548", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/score.json": "18237e32e2df8fc15a5df25c164f430fae7442bfdd9e308e9163de0902b829b1", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/result.json": "acc654181e9ca8b390874f414ddbda2bfeff30f49155595ff5a1f540a2f35204", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/postmortem.md": "562bbf5ea80d0aa877d4d07113b81707e24879da22576d2fb29a8aceef315c2d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/facts.json": "d7b832ccc9e606467d17f581ecfa8afef4255c6b7454eb920d466abcc26e13d7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/score.json": "f86f0f03a28736c939b52973e9503be1b06e1aa7ebf884d1ec04b9eb071d4642", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/result.json": "72439b3ac1d439bdabd23cb7bbab3d7d418193af38c6ffdf626ac75c54a00a4f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/postmortem.md": "e2f65ea23be97224652e6be2d7cbb3585909cfc1b4b6f05a5802b2184f0d8b5a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/facts.json": "2d03bd2b8db664da1db5bc7cc438bb8eb34d4933a552aac9c30db8fa8d60ad64", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/score.json": "fb6e0f41770531080e66c5ad501706e5108b9f1da138e29c3695b36481a33436", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/result.json": "c1f9ef10457633a7c18cbfdcd7fe9433d568eacaa5564206d65c74c64982227a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/postmortem.md": "a8a4f29a5cdd45e4ff478eb2cc419d84a02feac7cb0b419cc013d0bf71a87381", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/facts.json": "25d64c330e5eed978cbbaaf23ba5e739b5849c6624c666cf4cdc493997c79783", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/score.json": "4b3269b44480aa9473e604a63b393e14c65dec30eefcdb5ca7d74cb83eefa307", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/result.json": "f612936572d344b09f8cb4ef5a4c48cc8f07fb8693799c754af47127e1ad6f8e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/postmortem.md": "d7e081f8d2cba66756a5a9cf82435c2fd0529a3b779de7adbdd9b6fbd4ec0f65", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/facts.json": "32a411add81eea82b7b85f90e16d0993b0f4baf67ce2c26c75021e242af36d8f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/score.json": "6ed2e00856096bb423ca96d38968f9b4d250e28a65f94cc1a8d8bc9003e4625f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/result.json": "67a20bcb9ef6bf42961a3394dc6f59933be326c503688220b4cc44c0762cd6cc", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/postmortem.md": "5568bec03c9b1b51a1bc8285926abf61f16003d2a42c7f79eff836b1f7ba19d5", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/facts.json": "f0a32c0399d533733ff263641361c3a6603d14a2c5c39c6456bb16500317449a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/score.json": "d22489c864817228ec7e5face56d36ed1b1bed91bcfb0079a4f691253c07739b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/result.json": "6554f1c782a9d5c838c7b372be6f7181e86f092fdf76bbda5722e5f6da799ce5", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/postmortem.md": "e0ad45480ba78fefd481e127caa31cf242a9d1f3b618b25a471fb769c9f48913", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/facts.json": "71378112f6ec2417356f322329315d7ef33ad2fd7c4e66c2490bb632ba5ae213", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/score.json": "f3f2af9a19e03a24638974bee5daf5b5cd4b539345228a29441e6f8c462de137", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/result.json": "de7a7c27081a872ae9016af264ec30c14f185f7d58d1cdc4158051dc12288e76", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/postmortem.md": "0343118806f8543015e3945ebdff939369dafcfeee1b9ca68ede4c599dcd8645", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/facts.json": "e317ab0cb1c81f950313470a1f9ced38b8edf9490684f6d1e662fb168a0f479c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/score.json": "a6e75d859808a0cf639e67adb3bf80944cae049ed09e9d67a55157f1f18e2007", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/result.json": "b6c86fbbcf1723881ccd71c58a64c7e9b5ecbd8bad9c891d94a2ae35aecc171b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/postmortem.md": "d0be82d601d694a9feda146c6d0370c1b7ec427c46f946c2054ece4d26ddf2c5", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/facts.json": "489b9362763cd63a03851029288ac40680ac1964a81590978147a3f339abc4c8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/score.json": "c2a36ccd36e3deb02aaa8cd1e88cb3d9558ad30ebf67d6cbe24ab748ae71d206", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/result.json": "5838007ad7146fea7b5f32c7d7302f9f569833217e914beb4174b555916e2eb4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/postmortem.md": "dc639d0029a2a6ceee754aa7e375c0febe156fc58de310bea014bd323f43a628", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/facts.json": "ac1a22c08908eb8ea0b4b7fc4d84380f0941c9c5d57751a67bd3d0aa75a28161", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/score.json": "5184e12a5859dabbc3255f4ce8a3da612dab9dc3126a8ccb7b3c16e0bb3158da", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/result.json": "cc4a982102e477593d98add270a1514b653145c0369f1da80700be1a506372da", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/postmortem.md": "8c53f2337b522811e47e6fcffd18fc8d960cc5f84dd6fe8661417ec0dee64c7b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/facts.json": "da1335770e3f1e86197455729a9dfcfca19f532682c83a7cb71ab6600dd65ebb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/score.json": "3d67be9973645d365963a267a41cfa503f472edcfd4e0bdc53004aa3d90641a5", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/result.json": "342f8310977427099c3a18b00e05bef7e01d02f9bae4f86bc512578f65f53c48", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/postmortem.md": "1ae011246fc4d16ac373882cc1763d609ae3df104932d6a4f1637b02add9f6e2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/facts.json": "34a29339a486bbd05f32d5912ff0091b538fbe8c5664469bd07a0900d55deb81", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/score.json": "8a4636c4b68147e18fe65b04a7cd88a725ff295ffb32bf0e3d2ff067402f8d09", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/result.json": "08bd1a89a5ddbb93b3f85ffae598989ddc03c61d2b018d2c5c02620baab1a360", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/postmortem.md": "38e9f03f261795afe3d7f738e13e7e0e4e6d258d6d2dc743de6c9f29f1860aee", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/facts.json": "d678c7267590a8847a5d34a194bb0cb281828cb08decc2f26125c79a06e0a437", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/score.json": "f5b22843e26a8ed9e54313ff2c57b8fc7f0ec1bc88280ea5ea659f895fb62742", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/result.json": "339f47f88b9de3b14945d59725326eb0af92f05028491d7f4d715546ef4c2ceb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/postmortem.md": "f50ffeace47fcdc8364f1e97957b19de15afe3469f7b4ddde39cb97d203a7c3b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/facts.json": "826534bdc38a7965461e6b9b5eb000e585a60a23baae829e77d4d7024b7b445f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/score.json": "daebfbef9743682e5258495ca530d412f8bfe16c096ccbe5543691ca2d38045d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/result.json": "3e973b9ee15741f6c543d80f1d7d15b6795795e0a7ea56ff32397175b4c8083c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/postmortem.md": "7051103d7a6365c205178668b3baa896dd2196d7879bb15f368c2eaf4b32b24f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/facts.json": "8d2bb99a5ce1218888f5f9a68e57f8fb2d5e3c7b5e34f1c75d4b2f562711b30e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/score.json": "fa279c331ed9de3e036c6a46617926858002e5d8355184ad7e899940de3fd858", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/result.json": "43ea50488ec4026b59ebcd78fec51546312c6a904fad50c70018d2640a554036", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/postmortem.md": "c10eb30349fdcc65b4aa389a6efca1d5491b00fc8158c84334bba3dca70ed3a0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/facts.json": "69b608cbec7e0528581d3a366d0b75021b21fc2690f4eeba60646098b16d67c5", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/score.json": "39b0bb7a095428765b964f5b9c942dc0b56c41e27b967097ffbba905aec6d25f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/result.json": "c3fd3841c5c638eea2d85990d2f7539bddfb15ebc2169b7b9741c7842e5dae06", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/postmortem.md": "c7baa8c7d85f5a8a92b750205740a0636027daeb3977aff58b0c33a1b81d22ee", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/facts.json": "eae29c04637cbdaa0dd1e06576f670a58a3146f85358d2bc23afea87d63e5908", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/score.json": "6d4a305ef62fe3e75636ff21754974f53c460b7cf0be69917dd979cb1cb4ebeb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/result.json": "da751df451d7ba70fee854d048b081278cd4612ba32b930b73b7eb7cf451053a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/postmortem.md": "72c08b6e4be4047a8c1ede97e6dec30eb153d5564287b36123a9dc4a4bf72e07", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/facts.json": "0d8bebff6509039594634bc4fb616d911873c595b4f18fcaf5e89d0857d2cf80", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/score.json": "dfd4966643a6aa5779d2b84c90116752d9a1e84c30910df16743571d71fa7a06", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/result.json": "14e2988413cc1171788413a214f5f0423abe78a48cb89fc403d129bfabfcd453", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/postmortem.md": "a8ac22133e22ab8615c27ef271c9425fe68ec68819b353fe34bdf0fe2b6f3484", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/facts.json": "a5d4540f93e5a1a79f3ea6d9f3a7c33e53a18224d982ff6ded1a3cff1516a6ba", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/score.json": "b576a35b4524cc5d3d71c44567152929d5de7b6e94186a7ec324b2f60ffe1d0e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/result.json": "5089158f502a77b00831dd8a2ed7071035e2f99982dd83ecc5d1ac44ef484382", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/postmortem.md": "5afdf0faae1d224dbee6190ed29786cfb47d85b8103c31002b055956ed57e3a7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/facts.json": "78e9faf5a8a188305b92e37425a6d8bd51f1e3f30733cd600e6169c353f97676", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/score.json": "5ecb5a38d58d007ff6c2a8f3a74f47d197f9407951d9bedf8dc9f5d4823f6cf3", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/result.json": "c523865e9e62e4f050f6c4c41aaed88e209b5ba917b33321e47108df36a4b44d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/postmortem.md": "91dee5e71ca497b0c8a9d5c517e03563efb194f4beb5ddef694cb066be62e805", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq/facts.json": "2154ba1f94b9fd17fce742d45074d21e3e96bbf6f888a2ecf3e28821dd7b6559", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq/score.json": "25121781dbcf2ce2c22d2c3dff5f55cfdb514e4f7ac35ccd675fd4eefb59adba", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq/result.json": "563cdbbd56def7bf1be6c6fcff3274a409d355bfb5020dbda6d13d17b81404ce", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq/postmortem.md": "893719c96ce24a95b7b762bd5c4f8fcf19ebbf1cb770593e4a24a2b81a2ab5bf", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/facts.json": "b2463499d134d3e370c56571f84ee9f3e80fdee003086743ebdabd73bd494dcd", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/score.json": "bb6a1ada25575a17bc1a4ac3b353bad2112ef556717d0b1f62c6cb15bdbd7f87", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/result.json": "cceea50bf7406deebff4f57315bf4a42c434de55428c3e9e151cc6308a26736d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/postmortem.md": "309c78dc1ac2cbce0593b939e4e213b271e2fe76c8fd65043bfdd842eee26457", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/facts.json": "8242a3fccfd9cec6972f01a1bf0bea67bbafc906b73b2925950a70ca6c6f6d47", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/score.json": "72c0d922f2b14fc301326f536554cf894ab9741e206745cfa6860b992fd89a39", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/result.json": "cb0e553915ca352952225520ed0ed8f5f3064319f8d72d15cf9e6f5e47cee88a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/postmortem.md": "55cf4028d9330514ad3c119e6488c602f0c151db2c315fe509afca4cd6ed1a7e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/facts.json": "a2455bf3a057d183dd7c0128dbe9207112a3d5fff15925796c878e3e890f5b00", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/score.json": "733002ab10fd1d71ead3978997a284131339785870c67b762925bb8ea290443b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/result.json": "d0fa338f6f5887e876ccf078bc6305a63d30ec6d76b301619b70a149d5d9dce7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/postmortem.md": "2a8047015788d19bf030eb22057faeff58c68e7c6fae9b8386af06cd75a25a95", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/facts.json": "b4ffc0119f97fe02d9366fdc026bce234653b7c42986c7c25a4dc4d5762d44c9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/score.json": "c9a6c6e7dca5cfca14ccefaf9d517bc0efdaaacedc39363a17b9a7108fd79f18", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/result.json": "af90d2b182b71d228ac29b12cc4f37f12415c36dedf74e4f3dfc3994027ec49b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/postmortem.md": "5b34c0ee74d80eaac90e869f31d5e1ca006c3e9db782ea32151a5c4b96df4344", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/facts.json": "5be3076b138873e2665b004af6ad532c45d29284625bfe60b14e02d92b2cc729", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/score.json": "51be9db7a96b72b0732ed412d6db7a586a4c6deb4fceac36860ee64c584e9ec4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/result.json": "c94ffae9a8bacbb068952e65d5ee8f24a68bc6560565b566a7a8c50ad429b977", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/postmortem.md": "0d3ec8bf4b926d488c3afd430f15bda15d8fe2190bec5a88d41fb6782d8ca683", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/facts.json": "e300a58bafe9097224be43cd87bdc8c2f43f76ac71c4e635018484c77f03d441", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/score.json": "1bb4edb06ed944487c22891dc1fb89d71bc396f91abd9b46e192371834951efc", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/result.json": "4697026db61a9287de1a3249270fbe6b9d157ebbe10917f98e4880762f666d6d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/postmortem.md": "dc9b20bbeb704d97124dc3b3af0c4ac109443419e9e22e9f56aa433a197c47c2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/original-evidence-seal.json": "a0dea77f62813a27edaf0392f576eb3fd66b8398bc216e58c7e35c477cef1317", "evals/runs/saluki-native-0.1.57-2026-11-03/config.json": "385d5d59b80c7834284f670e89874618db3f48d683bbd669b0a4f6114d154a3a", "evals/runs/saluki-native-0.1.57-2026-11-03/process.json": "fa48c7dc77434335c60006be2c0716bb74659d2b6a99c83e34ec4dbfb28d4533", "evals/runs/saluki-native-0.1.57-2026-11-03/process-exit.json": "eba348e8b4bc3a01d88019c3a81ca699e225778f00b902dddbd43a420ffc943e", "evals/runs/saluki-native-0.1.57-2026-11-03/console.log": "9a27ba0cd786a24f4267eecbb4ed8121678700737c10e78f0a11bc3a832804bc", "evals/runs/saluki-native-0.1.57-2026-11-03/native-verification.json": "6bcad459b955e281d07a7b650e994b5a71d3bf43636384d228ec908e2f8c8b3d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/identity-comparison.json": "8614653a20f14356abcaa9bd3b540145613cf6e6821b0c9215b512c5913bec98", "evals/runs/saluki-native-0.1.57-2026-11-03/codex-inline-judge.json": "1ebc5a71988d0dd3f84acace67abf38688097e78ff6f942efbcb364c6970768f", "evals/runs/saluki-native-0.1.57-2026-11-03/offload-frequency-audit.json": "4f2a9c744b7b5f32b1888c6bd0be62e7e9811db82c325cf18e05f0dcca88fec8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/daemon.log": "d9c8e70e8e842930ff7b60853cfe24433b5cd03089ef81197e6d7bc89cd35089", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/daemon.log": "2739852e52dcf1225f4f36a076f1331a976e044221ab76385e69553a560dbda7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/daemon.log": "6b85593210bda8fae2ca90cc72348b008e373c1c9866b8ce97cbc8bf75dc0e44", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/daemon.log": "f92121e7f1ed292200899f493b79a76e106a609e8de56ab0a6df6e3ee776d536", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/daemon.log": "939ab1fdc2c127187bc12fa36e3acbaee11c32986097fdd299f2c62666dabacf", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/daemon.log": "908f8e729a1eba10c60b32273cf878c0284a4186be16933712db12f5573ea539", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/daemon.log": "ba0d86608e525cbe9a59bbf1a9b3d5531e98a334c95e7c12baa7116090bad03c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/daemon.log": "5bbe95f58ed499ffbdab0f0b7bcf0aa6fca2a94010f3f6ff5a988e532bc9d95b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/daemon.log": "3bc9b9df61939de2262e96f0fb8e8fdb2748501d739516527a9ed232203a681e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/daemon.log": "da49f109a9a81f8c31fd79e876d47cef554967c4aa4b127fd9f41513ff1eb471", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/daemon.log": "c678d0466b100b341c7d711542f146ac50844b6bf97d1ad212cab7418d891ff9", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/daemon.log": "988f191dc4d9056f8e26738fb1abcd187dd6442330400d1a3c532a11f8e86bce", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/daemon.log": "aeb9716820cf445c842b0781fcf988d696f5895fe070b25268a761768dd96ec1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/daemon.log": "68d41b6c80293fb56d50aa00654d19e50565e3bacaa2b5095de558c945045265", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/daemon.log": "eef850ff0cbf998ee5dec0c68e73d3a969c66763cdb55474912b1adf84171f4f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/daemon.log": "d362fc5c5a934fdd375f91e9cac16898a24e979e349c410f45b2c6e62006f6fd", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/daemon.log": "f062547058a5cc2c5ba1ca1cddab3197c845671cac31a2b513b005c13799667e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/daemon.log": "2df6f9af948b34b73624b95b85dbc1a813c76a39c1780255a4cded38fae1ee47", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/daemon.log": "028150d246d92a98d7de276bfad1e0ebfa47b29a027dc3ac73c05aa005cb08c5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/daemon.log": "b02df4f73265c7af93c4eb9c7d2669f31bf095a4a2cbd5878a4d61fd9b815361", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/daemon.log": "f2cefc1d04cbb0cedbf47949d34bd53dcdb4a31a8585c4f78619bd0848e8746d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/daemon.log": "6b48ef36b114e63ec09ce2a50e1ea6a6ca80f919a48100cdf4d2e29f70500701", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/daemon.log": "7a43785c39edf4b43be7141c3a1e70e08255ef4e16b791925fd9cd49b8aa5f6c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/daemon.log": "1e6a54fd34684c3ac4aa5d5facc28dd9b4e9b3167afbad7bc6b6a763e0e22eaa", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/daemon.log": "e493bc2202368e8cf0dd414960dba73316251282a5cdc030086f80cf852f494f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/daemon.log": "08c5dd0954c4f9272d3d50887f8e7c8c19985e50a05153ceb55dd9262cda42ad", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/daemon.log": "811490711af0d93d4b8f930c872ac02fc240705128fecf43e0438a8d8cffb791", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/daemon.log": "e067f702a33d5865370e1742df30f52f424f5c4669bd4e8a7a76ff0ede3fec8e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/daemon.log": "18e00dc492a66a007d6f4336daed7e986d712cacc2c49412b554a55d6385edf8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/daemon.log": "95a3b32c73529e91c950d4168b60fcdae7ce70a47428516a3c275c608e585e13", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/daemon.log": "c87b37e2450c5434f095150a8e1a0580cfa3cad30bb855321a457b66aa0c4b6d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/daemon.log": "e9e20b2370e6f4ed7d0b57033367007b3e7231931360a55c9ec9fb6be95dab09", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/daemon.log": "9bba3a4217e79f150fb68ba97f9dc190739ea7eba22a3837bd9715baf73a0909", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/daemon.log": "82861bb185a1afeb8e98b24e88da46c4e4b2dee81d1f6e55961a669da0066fc5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/daemon.log": "65aba89892cb75f4a4aa334ad03ed35f970c67a63ca07fffb9d07b6b5b593e6a", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/daemon.log": "ba93a0d6348398d75a4de256543167df188b8b06f94fadccccb27354184fb377", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/daemon.log": "722ba9efbc5afe43045d6878c95017c251e689f371ee515f142784c8b993ef4a", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/daemon.log": "f8ce50cc296fd02be07293be0c2ee542f06bc7e9bfe90b5bde557a1d4240c570", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/daemon.log": "e3efa8f54ca2dd8c6aa925380369b63d7ce20cb08c63d20346267a391607a8ac", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/daemon.log": "146fdfbc8771b54a46348181cb3adaadd66b9f2f06713135e3a73176d04b1cb4", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/daemon.log": "8c4185cc1d1b832e31e8f81fe15aeec11827a5389e51218721dfdd75d1f64c3e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/daemon.log": "64cb7256e93505d58d1007a9fe1987d3fcd6f18dbe6ea147d89ee548ad07f6f2", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/daemon.log": "efedb7f65dc05ebcff882d2dae329b03a31ce12fe48982d0a086f4ceee157006", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/daemon.log": "cbf66a2c5211345bb4967ea7de423faa4829f02f9d197f157001e9747b65c95e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/daemon.log": "dee4364c3a9470d44c92a88044b171d1b8bb44af8e860803594aa80fe4ba5c05", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/daemon.log": "1acab0ef8daa467bf2658cd6360fe69dbf68f81a861499eb01cc647518085a58", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/daemon.log": "e5cbfff8418814230ee26e40b810bddf0c70872abb54b15f4c880f505f7449a7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/daemon.log": "ee94e34976a6b59db40e97ba18314ec7ac7af8c2c5c41faa1c3726175fe20923", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/daemon.log": "2047ec4920a302c6cb079d1540cbb11abf3607ea22e0cbd6eccfd5471cb74d43", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/daemon.log": "be95babd6d725c84f7a479a261311d0013716ab8f14f395972504e0a7f2b8988", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/daemon.log": "5dd2a12c45aa8cad9c441e6af069a42ee5ff81a89d93bdb1a2cf40a0684cbb0c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/daemon.log": "08b3f4ad57783eaebb6cb04fe1a7cbaf26de9d4ab39c934291379f6c14df1b36", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/daemon.log": "084a42a18a650d78e828d3b6d25d279411a93dde1d4ef2b5b9f4724b69d42e30", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/daemon.log": "69dfe6a7d0c3bf1287bb4107143d24e75479935a81eb3ab6e8bc7bc9fdc20763", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/daemon.log": "9f37b45d34927bbcc4756bf0ba9b08dcd755e667726d47768f5cea9c023e7a5e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/daemon.log": "bfcc4d52cec13445148e7fcde506c7649d134f09a3b2c5dd21fdcc5f0913b529", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/daemon.log": "d9a5a62afdd95542fc2769b1a8b256ebc2fda5cb14e39a72880e9be58af3c3f4", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/daemon.log": "242afd3c41daa00e1646db67ebf1f6fb70948fc72eb57913ca9b697d1799f791", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/daemon.log": "97caaf3408e6d9b2a11ec610c5879235d4f978a7d1aaf2eb7f7c6e5161e6820f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/daemon.log": "b403f3d067589eedc77b911bf7953edfe7807c4eef1172250bfa0d01f8e902ef", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/daemon.log": "42334ebee64a3a2b7d049f89c0769bf9eaf1869ca43f68ca5376534cae46748b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/daemon.log": "56e707f88b935889d77cc455dfdc14eeaa1df725da0e70fecf479c098c53e27d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/daemon.log": "1ae28b958d2b747b6801c9f5d0e91e0ca8d9a723f830da0c97068174b3641a49", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/daemon.log": "7e0345304b8de61633fd6a1350107ce8246364d1722efcc81d4a0c23c7ad9ef2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/daemon.log": "ef3c4625ffb5a3ce70b974dc5ee1437eba7a6b03637a22fde18c6db93fa01812", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/daemon.log": "679ac7380e86b806c0dadb5aa42bba8665eddc17296580d32ba76f5f3d3314f2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/daemon.log": "82733005517dab958a631e5061dfada29baa706b3507288fe09cc31b6738b61b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/daemon.log": "be49d10695adad9357f39b2d7bc9c2d093d8d0ac78dc642e18831cd7041b9cdd", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/daemon.log": "d20c6143f3c51d0e123f710d546413f1802e9fa8271c3745dbf879ebb34aaaf7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/daemon.log": "4838fca302271993de3cb7b72990e49996a398985cd744f92e0820b3b0886e5b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/daemon.log": "028909902cd88b1f6e8f3c9f8dac46ded2e5f69d8ffe7d48869fd2ffb9125c4c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/daemon.log": "4c9dc0196484b017d66e5ca916bdaf8ac2e4f654ba3b7071843bb92697801e89", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/daemon.log": "8aa0e26ef954755e3617056160d155cf3207d43d35adce6a38f42cc408878f3d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/daemon.log": "5000ce9c756e00f1dce96c6f6eb71b8c67f0c4444f18c6bc7079e9e3f8688818", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/daemon.log": "237ee8e885566277064536504a86e85246801b5a539a9409ebe843003b4b8624", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/daemon.log": "00aaf6c84d03f0440d465011fdc290a6e2e10f6a8bc096e3e771599252936c51", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/daemon.log": "22c4ad5a1f974226c0046e52720182659b6272e1276e20e25089aa989565c0e8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/daemon.log": "9fa8bbb371ccec58920003138a9fc1a66b5a7a1b0c59537cd9e4e116e3a68aca", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/daemon.log": "7777dd53da733c66ebc274d298e99084caff923e535d020321c307bf0613b225", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/daemon.log": "065c4de582147a464a3c0518b0a89a862e1ae79940b26ef1cf36bf7685a0c744", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/daemon.log": "73628735500926ef349dbf2b1d894421e4f6e282e75d738e142c5fb2543f6bd7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/daemon.log": "cb0edb92d515d24df5b09f2afd242d9f0903f4d74f01ea1320210f3b25ca88f5", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/daemon.log": "1e04d6c4296236c4335fcdef4343984484ee064736b3d4e0e4e5962c2e86a55d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/daemon.log": "b711d6071e3655197caa814b32128b5f68b995844f4cfb77d68c31ca204e2410", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/daemon.log": "ae0782b5f1d4180744c5044ce65b9aee8d258265f86a8c1d160c259defe20b20", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/daemon.log": "ff2a4b0f9a8212b708eaa2af5874834648c8837b6f8293dd84aaec87e3f43616", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/daemon.log": "3a713ec8e50963a1587c4f14eae97daa22db40e8570d5906ae6c96376af58e25", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/daemon.log": "c9e9a047e62482866e84e8b8ef67b510e07d9e51b2b2cd73d790aaed0be3546a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/daemon.log": "34817bcefbf6c6f789ceb5cafc8b42c38735f7c2108b8e8a2fc3e9cfe70a8c67", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/daemon.log": "6983c06dec6e1b1afc12afe59e578ff188bbf43b425ae287075659281c995f80", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/daemon.log": "25662b0109144b99420581fcb08ac3f6b2977a6521e530459e4bfdd276c4d412", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/daemon.log": "6894720cd76f8e517c589b9767a4fb108bd3c57b314077cf188cedd7a4f6936b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/daemon.log": "ba511115c95c9298bc4d73030bb447fe85d37f4cd6c6d4c24dca46c37bee0753", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/daemon.log": "18285640b75afdc01144764fbb76b46da05cb0a66e584b945e8c0013d8cff051", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/daemon.log": "e9a37129fa0d975918e5034c8c612a352388c37501de5f9273b42f405cfc0787", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/daemon.log": "864bedec2e24eb178801c3a5e4a6f00eb24199a98bf30f353300a72878042677", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/daemon.log": "a77305d8edf32115cbdb8c4aad1898fe26ae21296d06b64286a6db62b6c10ffb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/daemon.log": "93cb3777550394534b39340356285fc9d5a2048132a428da0687d8be00b9c2ce", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/daemon.log": "a216f4d17d3708f39528e03c47b7669056aa56f01a8033312c32f68a4b313fbf", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/daemon.log": "131bb44348f9d1c4bc31217e9e5f1f9f62428ccccb43d0dd52ff2e546a227e02", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/daemon.log": "a12b613448f2ff4a906a70ab1544448af8b529899e8410c619f76ad27af384ac", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/daemon.log": "15dcb0245aa4bbd8fd34352d25b77b298d6fed78bcaa62a07696929040307e8d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/daemon.log": "3e786bda96ca31f726889c756903526b9e0b77437dffeb0e3cd48bfabf40e1ce", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/daemon.log": "05847f409418823445570e838ed5ef6eac64e02920e6a0bf2629ea83783b4820", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/daemon.log": "cec3d98e5ae0a7c427ccbd6e4fc6b9f9bff48a01e24ee69947b0c78919c4c8bd", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/daemon.log": "fee1af3ad1ab37848589100171f04fcc8b6eb0d958fdc5aa5c016a4ed854781b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/daemon.log": "8e3658f890ff56055517c64de690e57ff96585f34bacbf136ca57249d8eba4f9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/daemon.log": "a2647da17b3c89f47e5e3dc6f811638e3afaadce8226a2ef79575a45ffdad54f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/daemon.log": "5850e9639376af6ee2d13db6527b28f9e54237e4bc2305e63acba8d704a1b41a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/daemon.log": "9d24d28b57492dc35d98ad8be5147bd6d4bb2d8ff83384a174a878a8f1ed27e0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/daemon.log": "1c4a3bbcb7740274f9bf258efc71a9181f424568af626eb41f63fe5c65c5f36a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/daemon.log": "51f710e57894118e94c9ada7fc2b8c9de4c41f929fef850b24e2cdbef9b3a159", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/daemon.log": "083874f6bda2dd89518401a52ea6cead03f374bcc2a40c87220ff9ba968a14b6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/daemon.log": "3ea7e6565de1d6c23af28841af75f93681f30a4dfe864cb7cba641c1fc131ba6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/daemon.log": "a8000535ddb590d89b00108af375e837e94cfe8b7956f3166e5daa3b879d8523", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/daemon.log": "ce3ecbc3f2329a7e4851979c864b2c9969b90c324f63b0ce3bb6989994b0ffa9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/daemon.log": "1eb9574cbeb8a112d14acd4914c283103c46857c1ff1ee7872f3c1a3ab3a6efa", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/daemon.log": "356d89615f012894963d5f00c9cea4580a7e8dcd0eee4e1ba3201140b46a065f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/daemon.log": "2099930699c9b7dccbf3ac4b7d7afc393ec0d9cbee9cd0fcd19523c3df82826c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq/daemon.log": "db5511124368eb487e458e2b402bbc0147e9b16dfede5a26c16d4605ac853f2c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/daemon.log": "170f995578c0c2ba86f4e152eb1acdff0f1c28f19b270741e9b8645e44df59c7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/daemon.log": "6ac18709b905d76be67fd3f101c10ae72407602544d2bd4bd09e0d615741179c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/daemon.log": "27a97a2df81871e6c14b48f9c203fc4f0d34dbc5277dc4d6690afd5062194fbb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/daemon.log": "d6a24b70af5b002210df05159c71ecd10492b844684e107c358eb0778f3b2ce6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/daemon.log": "10cc46d79fb28a8c37d479a9f4ee477b66818920d9e88b3639c36b64c137a388", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/daemon.log": "e31a83c465f0f53392fcf60edf57117079d87b28c7b21ba5dfe52711c75d7b17", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/daemon.log": "9d6d340d90e2162e266a99e464cfc67b51b1bb62da9386d6ae0233c1d29e7cff", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/daemon.log": "8cbd56f2a12edd06c989c549680a3c589b2249c2cd23d3a398b3bb745528de15", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/manifest.json": "eca2686b3246f79b65b605cd17d1f76486c07bc8e6b68b6814a55197667d9b33", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/summary.json": "45919ee727e29ec7bfb3ae34c807937f176512653de16141e23f3d55a349caa0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/001/review.json": "34a44a0ddb61769defd317ce64639aa98e87feb05702f66aa9ed775aeaa4f305", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/001/prompt.txt": "cb35e3d088759a4362406e42dc27797a7798258d37e13820807b5b3301f5dd6e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/001/evidence.json": "1f8789319e361dd3394780fb421f742ca00f7bfef129d7b2864d434d90954b29", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/001/cli-output.json": "3d7b62c9a2ba85f425eae069ed910033e2b39f5522e8f33dd434a65953fcde3f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/workspace/tic-tac-toe-game/index.html": "e9b569395a328350da51419c7b119c76a20fe7c503d378dc05108b34dd5163f5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/002/review.json": "25c6616ad53aba89618d184f887dcd3dc27038e1d9bd00bc191d2390296bcfde", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/002/prompt.txt": "cc515a209dbe3e6f4a9f39867c2321a247aeac973b3f2a0187b33778ef55b47e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/002/evidence.json": "92e0d520b750c61400207c80f8919405eb083b0ae79420e0cc016b61053af12c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/002/cli-output.json": "dbb8b1d9603f1479e8f04e4c569eb2ab4ef113813e83ff0d91e8d9d72fb30146", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/workspace/pet-shop-website/index.html": "74b854669c8c13c12f5a2ffd7b8709db645ca1506eb37953445aa63d0eda2dce", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/003/review.json": "672049705698e3d713fafa0335747b693458211faa135d232bfa075e64c0ea8f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/003/prompt.txt": "b95535e1e21d6351f6f3cbea5653e2f4ff22cb0e52812713de0e203bb3c300bb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/003/evidence.json": "4726ecda49c77f177066d35d232c6a98a20665aa2cde9b7115b4fac34a602eed", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/003/cli-output.json": "447d5954de3fd0eba0ff068816d8b132f10aa30645341686809c92f226a718ee", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/workspace/tank-combat-arcade/index.html": "f82a8b63a40808d8cc75cfca7c4f51e2c1150ecdf5b42521c19fbe19088adc57", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/004/review.json": "1b227eba4bea8d0c13a80a1ba4ff6bce399a34ea8f53db82a4282e31ea36448b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/004/prompt.txt": "5d4ac2ad3fbaf98e15fdd9b3699a49f91d9a17ff86bb68e6f5e65fd9e1ed745a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/004/evidence.json": "16a597e83116a264dcf2d9b1f6eea59a3b05047f47a9e7350ab4a0cda3e24322", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/004/cli-output.json": "353f2dbb2a01648fdc41b8e4e44d94180d6fbccfd295a633dcecdb8118da39c7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/MIGRATION.md": "697739fdfeba053e051f9819ee26ac2a1c9a01582e321f3c9f4d82a377a115f0", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/package.json": "da7cfa9ba7722e4bdeb55cc9b373f3a311dfd6096b48587df9fe10286cd581fc", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/README.md": "e8ad3a935d40b5659206cfec22cc684ef10b2e5e673130d9e501d192d87d1194", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/src/handlers.ts": "f151082616220d177a6e1bc1d525727b820376df24996db8e901924a3d2b1b85", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/src/migrate.ts": "e9f88335fc8ef947281f981749163f56d5480c61adb692433e3790de30a346d2", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/src/store.ts": "6c877c17a044a87df7ed7af1488eb1f879d841902fd183eba20260a8fe34c735", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/src/types.ts": "64ba55edba66e2e1b6867c13b0c25aed688e080b5546141a031606743070b694", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/tests/migrate.test.ts": "69aeada66d159a38bf67ea3d7129c8201394d56938c1a61a84a727edce09e818", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/workspace/user-store/tsconfig.json": "4d432e97e14419c8cd5438b66f9177e84275f76e029c7f13d30683d65fec7c38", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/005/review.json": "6485a13e47b52ad88f37b3648d87ba6663a6d6d8813e6f0799805bed9dcc4788", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/005/prompt.txt": "b5eaff4b123cbda2ed1164cf3b48c7bce72bb4c440ef1424080d00167b5db433", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/005/evidence.json": "e62a7e7cb191d52fb800d3134dbc26ba556960f6fe24a5480eb6f77a58e17aff", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/005/cli-output.json": "c83c8d1d75799ddabcac1035550999db11811928f1189021883cf6a3ced3422b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/workspace/machine-kata/package.json": "6f3c6e522efed23d503e1270df18e6aef4f87a411c7be2d78cdb24842feb392d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/workspace/machine-kata/src/machine.ts": "126ab4f1e41d0ff81d190c11370c35f30a30cea15832196b08012d359e3c764f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/workspace/machine-kata/tests/machine.test.ts": "12e2d36908178da95c12c9d5bf5d3bb7d8c67606f2ec4faa3ffd5fff62517997", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/workspace/machine-kata/tsconfig.json": "8bda3c0260964f6b5352b61969a681b7760822e954787a150c0e959839d389e3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/006/review.json": "1f26f16203ec7925e151f3b201ea4b5b418d6662c4dabbbb81cc137c9f7c09c1", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/006/prompt.txt": "726851a23eb5f50ccb4b04e3b6fd21b74d3bfebde7484b0b00d5ca15978e21f4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/006/evidence.json": "3a47336444ef429637ee70e4448fe2305c05db14dcd5c9b476f53cd28bed496e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/006/cli-output.json": "83f56c2b64a9dd9aa223597e8cc7b8ebf82a8dabb64af5bbb5a033e742331169", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/workspace/pagination-service/lib/paginate.mjs": "e534c01618f7c0906a7b865e21cf9bbee21e313eaaa8fb02fdd14a2b078f3353", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/007/review.json": "49058df9928ab4c0d7f0528e9bc648ae5c7e4eeaa6261f4672acd898503f4161", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/007/prompt.txt": "ab0c7b9efe390b1155b670c302fb2e50a0a169294812a614bb54a37dd5e9dea2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/007/evidence.json": "4a3fe0f7b2147cfa253063e6a689fa7f509410b52652ca90c83ffb584669982f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/007/cli-output.json": "3f827c1f8d7a0dd7a5848dc78e7b13a5c5b308d9db3b9181211fdb652e860bde", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/workspace/customer-data-cleanup/data/raw/customers_a.csv": "bc92617373571bdf0bef1c29ee05aad44a7a68274c0f7334b1ebe8aa5eedfe54", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/workspace/customer-data-cleanup/data/raw/customers_b.csv": "ff167222077dd5a971ee88d6ab656d98b2b2051e84ceb223bcc893aab9186b09", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/workspace/customer-data-cleanup/data/raw/legacy_export.csv": "90e6abcfcf404de96b3ac6385e3ddc3265125552377f3c3428d5636d310f9afa", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/008/review.json": "ece50aae7756ccc13d23f1a005d4373f6759b94b4132863cd6d766800ee46903", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/008/prompt.txt": "ccf84123c95f7cf87046211d16812bc609b0c4e3b0b63426b45aa73d18d6392f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/008/evidence.json": "6cfed4d266ec6a84655ffca054f3d542e53ec56f63e5cfb31a753691e9625357", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/008/cli-output.json": "79cb7781027ef2e664c05cf3a9929c5e6d5312dbf4bd6985321aa7473994931b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/009/review.json": "8c3625f16dddcd60b3a729c2d3289ee9eb79fc995d2cb68fa925494f94b54f5f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/009/prompt.txt": "6076a420223cabb4a792f13ff52743eea505fb9b7cf412db4cd640259292fdf7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/009/evidence.json": "0b97785d282c4c9e846b90c5fc556102c6433586fcf592da4d495578a295bda5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/009/cli-output.json": "21d5d73e47a65b148716abc48283b101ece642977bd172c62fb5772487886287", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/workspace/quarterly-maintenance-window/halt-report.md": "e04f62b6bca209d99b805ac5da9ccd418dcbe2833ac55a52867bec315a7f28b4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/010/review.json": "fde0992945a02fdeb590f1c5813503cd26a8fe52a5c9ee7e72310573e545693d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/010/prompt.txt": "b86ee59319b8edea5d182aa75fb8e62fcf03a473e25d980f27909cac337978cd", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/010/evidence.json": "c25b0a4374b938864349fe651ac8b8f78b8e175ac1c94c149a627e43f44b6248", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/010/cli-output.json": "bbe26d2a5f723b7f61120d18157036e4ede826e148e084bbd4742d7c30489bb1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/workspace/harbourview-office-relocation/plan.md": "b337dd9a43307ce859b08987892744be93ad8ff0bfd992e399412b68b8e19988", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/workspace/harbourview-office-relocation/brief.md": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/011/review.json": "503cccfe3b5674560fe46babcd105fc04fbef0a336882f654fb11d9d318f105e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/011/prompt.txt": "3493b9e59b4665d11eb6e5d54d441f73bb14d1d5b44cb58c5eb2b67a9186f1b3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/011/evidence.json": "b44e6b374858c727ea51dc9c4ef08fd353df6c0eb85e7716112bc8699eebebcc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/011/cli-output.json": "f12593311fe69d9d72edcb009778f3d506fead941c9c415cc90b2721d2a0691c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/workspace/skylark-launch-synthesis/synthesis.md": "9804e60b3abdb16414ad91af6dff919b1bc3885adf47d01cece6d6e7a8e3bc19", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/012/review.json": "818a2bd103678763e4e3da3e04cc82f400be9b95ce2ce0db546faa3b55d06554", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/012/prompt.txt": "24e75a518588c97ca582534fbdc8b17ea863f4cbe4930b02cbe9f22b0ad7a07d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/012/evidence.json": "66893a8bdff5e7e8aeecbc5e7a693498bc215d095e01382ed29a562e0ec26b88", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/012/cli-output.json": "4c8c85d3ee45dedf44c540af75a275aa818df0e9dd8dfc71014b3fd219b34f22", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/workspace/driftwater-outage-notice/customer-notice.md": "9b3e78c5c6b3288a92ecbcaa1470e2964c31cd783b12d4b6b9d42bea2a40db57", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/workspace/driftwater-outage-notice/facts/incident-brief.md": "a65cd16d09c63d27126363a03f1bc664d26e4c6fc58eae192477d98ac3d02251", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/workspace/driftwater-outage-notice/facts/legal-requirements.md": "f15d48f9a7a37a2e3d855566ffd9cbdb9f0633a9f8c7f0756816b40a15815920", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/013/review.json": "90b3ba0124ea38db8d5dfe68ef8d87ba900d50a83f01ee2cb6eda6d9132e69bb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/013/prompt.txt": "0972b069199c3ee08145c92f76aaa9ac5e726d27e87359f1988aa98b4f35c32c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/013/evidence.json": "b13e64f256b5050a0246ca18e99a87b4b20542506bda1fad39cd912d2380721d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/013/cli-output.json": "8b6533cbcafa1682db25b27576c9affbdc846bfbcadb78787dd1b1bcddc531e7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/014/review.json": "0abd56086471d5d5ca26d9257a92d17710f73e24a92beb592a3905fe650041d4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/014/prompt.txt": "50125dec6b5947ed796979371b69bda2f13353f0205e23104c56dd17f45542a7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/014/evidence.json": "4d1eb7fbf94eb2a8ed453198e5cc5e45d5192252ef17369202169edf839a1070", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/014/cli-output.json": "912e013a2ebf72594823c6d28cef14afbf4854ea6e69194677803234dba0ec0c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/workspace/harbourview-office-relocation/plan.md": "fdc179aa734adb91380ca44567c2b7eb768b64f8ed51d07c7aa0100b9c028d72", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/workspace/harbourview-office-relocation/brief.md": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/015/review.json": "00286b91b5ea4d9bf39ed2fd5862da983a9dec855d3f279a6ef540da275cc575", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/015/prompt.txt": "773270cf79d480481efc484178e3bb25bcf202a95735320a55a49c11e4022446", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/015/evidence.json": "bf77ec9d7d293d941f7ebd8ea0e8e4081e3e47b73d7ccb365265883818f0f385", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/015/cli-output.json": "ef62bd92f032d6841e4af6829d1ad86654211fccfbdae9c1ca99a2ff011de271", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/workspace/orchid-launch-follow-up/meeting-brief.md": "1523b6e14ee577164ab0371a70a01f5e296b2c9d06641fa2329c0e3aa51276e8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/workspace/orchid-launch-follow-up/meeting/old-agenda.md": "8a895418b408627cf5fd7367a45341bedc5fc29515c61dac05de3b47f6fdde38", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/workspace/orchid-launch-follow-up/meeting/roster.md": "8f77db20d7700521aecdf6cddcbbe807ebf99bae5c2043a558fd5f96831cad69", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/workspace/orchid-launch-follow-up/meeting/transcript.md": "7a4039bda8b643bfdfb3f8d1208a5c7835fd3b13f280867db9b6915992e0800a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/016/review.json": "15334f60719685453e51f0bd49ef865fd41ddcfb0df5853b77e01fa56958b37c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/016/prompt.txt": "4f175b3a82694e2fb1a37cf64285501846d02539264a371c147ed6d64ece6fb9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/016/evidence.json": "f3f4d6a8b895419f50b6a1fe4cb91ff726eb18c0d1adf02bcaeaaee757d31fce", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/016/cli-output.json": "7c01121c45f23fa7c6ae4bda12a1786e2f6a6aef6ed15445a8e776158744a44e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md": "757cab1e8f9a997d02a06feb526df76cc8c2660e44c7adb4aef1d6650536bca5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/workspace/spreadsheet-financial-model-workflow-eval/source/brief.md": "9b7893b96d2e07d87b152dcdef1c8e16395615b8bc827a34cc10dd0706ab0bda", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/workspace/spreadsheet-financial-model-workflow-eval/source/craftbook-eval-brief.md": "6cebbc3d1462d96e4ee669842d5d1d5fcc2c9e0c95bf6e723aefa688b297cbdd", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/workspace/spreadsheet-financial-model-workflow-eval/source/records.csv": "430befe636c28a4b2751e842a79773440af0bd7a5f1b742ea177c6ef204d74c6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/017/review.json": "bc1100f0c6f4f5ee57f9305f38bb791adc3ef6c85f3489a77f7e91e593f735b9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/017/prompt.txt": "e989b45651d5dbfd6da9490af2e7d60b316b5d08213d3ede0b52ec8553312d97", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/017/evidence.json": "66d196587b567bd25313298eb5792d8b489f16efd3b539bcf84c777baac532ab", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/017/cli-output.json": "718d4f6ace5d764894eda213d5d669e605a1458b07559f9e5b59dd003ced852e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/workspace/skylark-launch-synthesis/synthesis.md": "53f47ac9403de30d733875e79d1ed1a1042e87e7e5e592abbf9d4465036465d2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/018/review.json": "ad8f58b5c63d712580fbfcd0ba039be801b9c6459807c172dc5e99f558e58745", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/018/prompt.txt": "08ffa8f89d1b922dfa0b1cf2d6bba46f882da9c7c271b084f6b20c717cbfef21", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/018/evidence.json": "b0d11d3b7239d9d2c72e402e67512b96a478f0c021a1af73ece56dfe5ce140b8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/018/cli-output.json": "1f75cc4942543eae15190cdc1db6dd8826f05cb9b5d4c9599705f83142de2bd5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/workspace/brand-theme-alignment/theme-report.md": "efb76172f12d8bb545ee210e1a5aabe8099a82051849b0dcfbb4315e8a3dd703", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/workspace/brand-theme-alignment/source/quarterly-review.md": "70a119718fdfd89aba5d0dd2bdc2e7df8fb45f749afc71d56bb5980e1fb030a8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/019/review.json": "bdf9d76a12f72fdc4c2e35ff9d9ea4625a1b2ab92c23f0760308231bf5203a90", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/019/prompt.txt": "b20af1ba34fd1b509de0d205fc8c0b2496b2940f308b563ebff8d7f8dfaf8ff7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/019/evidence.json": "cb8165bb801cc87395c700761972c5477ea8b4fa361625149873a82119a28153", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/019/cli-output.json": "4061654bb8f5dec1805a6e85b94f17aa5b953dacd19f4c64a5915bccaf81af3f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md": "d1d494f8904a0462aeb19b8fa6fd39fac72008a5b4b44e1bf41b8929795e71ae", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/artifacts/powerpoint-grounding-eval/tasks/1/sources.md": "d42ac226eac0c9d9dec8ed539839da86e723f88a97e1fed1fbc284c85334563a", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/artifacts/powerpoint-grounding-eval/tasks/1/outline.md": "bbf7ba82bebf82bbc6112503324dcddc37e5da56cf099cbd48039583bb2680cb", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/workspace/powerpoint-grounding-eval/source/brief.md": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/020/review.json": "fe61f8d1df89e99a01425903b7ef12b39ad49da6d3d15b39c057a097701ff4dd", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/020/prompt.txt": "8460f284dd9b51ac146fc5db48b77f2817af12cfd30fdb9ad96f050e6b1abc99", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/020/evidence.json": "d2a130f3ce085f732dfa6c97687034438d099d9381a06333f140f24a1f65643b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/020/cli-output.json": "4499e08c61e69b74432219cdbcc03db16e0101930845479666a189b8f1ccc037", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/workspace/analytical-engine-research-brief/mocks/services.md": "4e5e67d58565bae7556a857a70daeccf632c69b60e3f581d5f46628c874cde66", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/021/review.json": "fcdc13b19f0047dcde718daf44903a4e4d67a8d6f73f952b2b2b7cc20f82c84a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/021/prompt.txt": "4945e584eef46e7187579430c76371a63f6e23228d79f4162121c1440f72c628", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/021/evidence.json": "6046f4dbdd7a83cc4a73c17f8ac5131159608734e25a6ad3747f7c79f710ffc0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/021/cli-output.json": "4bd4c2ca214db8085763f042ef16d9ed97db62b8f3b36c24889de19bc8d1012b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md": "2c51abc9dab8c0d42b1cc16a33fc94ea9e4752342d5a8d3f6f5d1bbfdacd2d56", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md": "2c51abc9dab8c0d42b1cc16a33fc94ea9e4752342d5a8d3f6f5d1bbfdacd2d56", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/022/review.json": "2ae985a076069d8a41d02bb69c3a85d492a841e33503a51110de3beac5ba7113", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/022/prompt.txt": "81ac839dcf7832a05d51497a5a652c26e5d87865149f0cba4c69741e7869fa28", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/022/evidence.json": "d417dec1f02676bb6dc7aac5b1bb04370cf546067c229297248d51aaf9bc1192", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/022/cli-output.json": "bd01cd6dcd30d3cefc0b3093c6927954c3e315e1782fe1486dbb3c7955052bde", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/artifacts/a-b-test-readout-eval/tasks/eval/readout.md": "e7110b60850807f4b678ae1ed7e61eeeffacdc40b1b3f8123594aa6ad1e36f44", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/workspace/a-b-test-readout-eval/data/checkout-test.csv": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/workspace/a-b-test-readout-eval/data/stats.json": "4de153609018af8d546b4d505836aafe96a5463092b21791afa8ac64e42f5774", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/workspace/a-b-test-readout-eval/source/experiment-brief.md": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/023/review.json": "476c57a8f097eaa78318e3b7be7d7af20459f04414992e5b44bd80991c47f742", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/023/prompt.txt": "ed71c9eea7c26ab5fa684ea2aaed8f39e0dc53aa54e29115227f01bf2829e393", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/023/evidence.json": "bbdb858c77d880909558ec85bfdf3d9fcefa62f63a6bb63d7099a9fa1d5bb3b2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/023/cli-output.json": "6d817619e571218f5cdf5c4594411f578331cdcb15c877954a3023c14cabd5d3", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/workspace/annotated-bibliography-eval/bibliography.md": "f8b32bfb911d130c0f3dd93581cc753070de5b01b14241971e9378ec0af1f1b7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/workspace/annotated-bibliography-eval/source/source-cards.md": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/024/review.json": "31910d6583d7a40d54873609f14b351f2164e8657257ffece49f4360c539d01a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/024/prompt.txt": "35744a102948054d4d2a148f3c5b94da994a28ff4969546d552ba1cc751083ed", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/024/evidence.json": "075365c075381275a6ad65cd8defffa12a7953d800c81b1ef973604c43b1ce5c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/024/cli-output.json": "9c4769bd730ad40f59a3b0eafbc329e3893650ed24844c3db4569aa1c5f5c07b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/workspace/word-document-from-content-eval/documents/eval/report.md": "1750b5d88eab54586917e151cf8d029a46bbfc6590dc6dbbfce25c32817dad1a", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/workspace/word-document-from-content-eval/source/brief.md": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/025/review.json": "53c915b10a77ee2be03835536f8837f238b62f7fad2435140039e9fa81064fe0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/025/prompt.txt": "b9d0d4920e09067c823368552905b92ef33595b5b44e4c9b4ebf41094c86c1b9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/025/evidence.json": "8b3a6a80a5650821002d105c6d3e13cfca11532db962f333b59d96bf2de94ddd", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/025/cli-output.json": "506cf4d1ee672f25873a4967a34a0dc6767f969b41e5809cb5d3063163c9fb6c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/workspace/squisq-core/packages/core/src/spatial/Geohash.ts": "2a0696ebc7b8c08fa9f0faf52cde78c3d7a54f7caf37ab6aa7f5f60dba8eb41e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/026/review.json": "a54362e2d11b7b0fbe5e0d9f7330fc5c00fa06a92884927cd726b206a0ad5afc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/026/prompt.txt": "5a112a652864a89923d9de1c0dfd533846e98177d89a772a31c21fc3bfed5744", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/026/evidence.json": "64c9c8db497879f22f0fee5a0b06f8cc120a0c823fa10ab8ec7634778f562411", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/026/cli-output.json": "5e82e117ee072ec59a758a498b79abd2f9735a3d655d306fca90f62d7983e1c4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/027/review.json": "b565cc81475cf72b910f61327f0b8a4bf8a004fd3e76461757c124527b187852", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/027/prompt.txt": "2b4377de31a2e0f684c474b075686c0e42dd1e0107936a9c438b5625507a0134", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/027/evidence.json": "6ce4b54b83bba30dc1b2ecabd42d8594c71b867470d21943d92d4dd5fa60c8bb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/027/cli-output.json": "f099fe547461339e07fcecba9e580782937ea9738ac4be0e4621aefac20066d5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/workspace/event-pipeline/src/types.ts": "0a51c5abc6308e5cd4ac3063688969109ef73eb296bd4c4d1e5099751da0503f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/028/review.json": "e97b5a6e09d6efc1a89ee9f24452a0b075403a856df1a8bd9ed6071942d742a3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/028/prompt.txt": "f91105a88f4f72bbbf23626c99204ba7dc5a4de65363e6bf291118101c21a881", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/028/evidence.json": "6c13acd501ef05df1d1273d31108e647646db452cc9b90001b56d570f468380c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/028/cli-output.json": "60bac093260613225dcbb4f95caeac8302a6126ac1ce32428515ca5ff5dc3bc1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md": "6329ff05a8053844f688a7773c17b3a704e30f0d58f950225280bf83b2023507", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/workspace/code-review-workflow-eval/source/craftbook-eval-brief.md": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/029/review.json": "1731a260aec509abe9f96681f5fb94c105b4b667cb45e654ac0d6ec564ea6727", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/029/prompt.txt": "1a18d4c473ee81806c748dd2f3bc9d4d8a0cba09a216621ab588622e910897ee", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/029/evidence.json": "66de4683d4e15fef9b55d289dd77584eb41641eac3e6d0ce1a1b1b4172660027", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/029/cli-output.json": "0ab6e99fc66f92978265a697cdd160b7ee5476d54932839805472fd230ac2a48", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md": "9e2447bb4db000c67544de820b409f26448ce4664d893c9d58993e4010b781f9", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/030/review.json": "8732f1dbde6dea55bd57f5d460845b7ca98ca2f1cdccc441fafb12e277442c52", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/030/prompt.txt": "0882cb4ad8676329759838add8e5e0ab2d8f8831b0617cb0b7bffc0882830b7d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/030/evidence.json": "a102016f36a43b172e99bab6063d903d7d4c9fc4650c91b012423572c65d956e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/030/cli-output.json": "5077a75fe4101a63af603794f43a73083e314576ccde8d40ae620800403491ca", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/workspace/security-maintenance-eval/security-review/REPORT.md": "a1e841cba3902981dc32d44da473a6b832d9c29f418e385bc8d3b7106a58b469", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/031/review.json": "7528ea6aeaa56cd1dee6ff3d31dd63c5037c5601a5558ef70b3ae3fb9e2aa8d4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/031/prompt.txt": "5bafc3f9f2b50b431fac8d5091a0eaef063dfc667029a7564d010f37a76b73e0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/031/evidence.json": "1ccf6d6bbf6e3250b99f3d12ab919182469402ef96dc394b0e1b53ac0f3fa1ac", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/031/cli-output.json": "4ee15934b89797be44b342db2804f647a3336c30fb82da7f6dcda143e839a3d0", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/workspace/large-pr-review-eval/pr-review.md": "1c71327ff54f70166e6a640f393e58c9f01c2d538ff8171f756cadc9b5b898ff", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/032/review.json": "a763472da42e4a3e6638ae87bec724e98b204cf6e39bb72da67be5ffc5de1940", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/032/prompt.txt": "ec8d64742d6144d7de225d4ae80415824d5b931ea19058570fbbccbe3f51bf7c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/032/evidence.json": "f7002983e1657052b848d3030a8e28dc25cdd60a1600a0a318127182a2eb8d6f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/032/cli-output.json": "580f65bdabbc2eb817aa84fc2e47ad97775d1b480219b7ccb2a68eff852d814b", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/artifacts/profile-cards/tasks/1/verification.md": "f622bca054d281b15d56713eee137dd9b0d484cfde085ff4964e3061aea38ffe", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/workspace/profile-cards/source/rename-brief.md": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/033/review.json": "b6b42d3b1de902c41565f4d2a58fb07ec6f8338863a2c522faeb94dbd6e1f7c5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/033/prompt.txt": "3b19f8c9f065e18328b3669f3108c4c6d2146318bd050c3495e4828d6a4449bb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/033/evidence.json": "0772f10b3f7a9c90fde50edc1b55735d60dce96c9c54d7fe7d44bdf71d6c9ed9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/033/cli-output.json": "3db3d07597586acecbabd9713ee052fd26ac719ea280f878a206343f5151c3ba", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/artifacts/crewlog/tasks/1/change-notes.md": "69717175f34a2f3ac3856869d31ac8508729ef275ac80308a05184ec8a912bc3", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/workspace/crewlog/source/refactor-brief.md": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/034/review.json": "7b747120c2f31c64123c2acb35c715845e930cc4bb47a737506b45aaa33aa41e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/034/prompt.txt": "3350a90b60913866f5fe5d12291b993e0b1b68e5fa6a09f261b66bd0c8ece363", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/034/evidence.json": "0578b463c2a64859cb85c5d67c2665cf913a9cdc81ef32484e9a44125cebb0a2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/034/cli-output.json": "9a57424e83c746fdff07d3d8edb432c829aded774b27642ed093143102eeb381", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/artifacts/orderly-pricing/tasks/1/fix-notes.md": "9d2b819755b277242067b7f75df80e9af8dcb8ad4716b2b797c2d72a858a353e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/workspace/orderly-pricing/source/bug-report.md": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/035/review.json": "d706ac0c67aaaf60105fc659a0fa9c383056dce0a129e0136e0bbf6da8d71eed", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/035/prompt.txt": "a30da9fc70965aa4f27a319502ada2a719deee5d346b1f91e347b90887d0f476", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/035/evidence.json": "ab916625d1e1cad681d30d4578a5d7d45b73d32aa284d63c3d430049cf9b373f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/035/cli-output.json": "2370a518b3737924e9672be58c8f281dcb47097c200c30429abf35d7b192f73f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/036/review.json": "00c2ebd20fc035efaa4386f9c8ee57f86ab5295be7e4b764d223509afc7d4515", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/036/prompt.txt": "2c10e5917868a096fa03fc79dc87c66e94f932e58bb55ece42bd215a0c4ac920", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/036/evidence.json": "6e739cc20031c3118e4b65d49816f312b8694f94164bd4f4a7ac3cd80e2bb46b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/036/cli-output.json": "d6edb231849823d483ec970520fda7e07a567532d3074a6c4f055d8bfa0c5db4", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/workspace/quarterly-brief/out/summary.md": "4d8064f3dbe41999f9a3cc454938f72c55263e6a6451df3ab290aa56319222e9", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/workspace/quarterly-brief/data/notes.txt": "949c911a4bbdf1083c68bbe94d539edda30128a353f4028a07a65bb717752a47", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/037/review.json": "4b828e90846e8198026e32e2a8eab2396628e985ece77bcb7947c47ece28144b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/037/prompt.txt": "fb9eb2190a767c36cdb9003efb1ddb8586d083187ef18494c7ab34ded8f86cf2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/037/evidence.json": "840fbc475b32b048f223365f7114fa9a3c599a948b87b564f9ca93d009d09702", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/037/cli-output.json": "6d5f8e3dc993bd80f9c86f6b1b3522589172ab26930e7d38837134f6e387e1db", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/workspace/fieldnote-studio-office/report.md": "a84ca7d325df51e97cff8ec11b8fa46375e076bb86ffd677be12dae935d56c00", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/038/review.json": "221deeaa3b9a411ab47db21fcfddfe36f79adac2195d9edfb9de275b1c335369", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/038/prompt.txt": "1aa35e2f89f66396fdedcdca2f45086e7449609e59c28c726a6e115d650f86c4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/038/evidence.json": "ca0ea522dd0cc01b205872a9a37d98639db2992807def8330feebd37a213014e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/038/cli-output.json": "b7322586b8d81fe429526a9773039aa680f908d2e0ae672bdab837fd5c8f65aa", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/workspace/order-intake-cleanup/out/report.md": "cb924436ba6edca4e9ec7900c10eae464174b7fa5057abb8d0787dd460a7e985", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/workspace/order-intake-cleanup/data/orders.csv": "c88785ed78c4d8b14484d8d0f04516c9c93c99045b2e90b60b19f96ee17d7bb2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/039/review.json": "d76fad01db00842355ee5b9d353022588664f2ce82dc3d1747344d2e4ece74e8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/039/prompt.txt": "5a0c25d112f8d64f32e1973f2a6b7d9b87a52617f698de4cf31ca366549a8e17", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/039/evidence.json": "5e42a8d798687f2c2b589ae2a4853f2b11202bd08e609ca30b4afa349c4a2a8f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/039/cli-output.json": "3e138606e0d28c4041d9d8cf52bcd917f2870654b0d401a09ce725e9d29bf44d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/workspace/fieldstone-launch-comms/out/press-release.md": "1e5839e497f08ea3e74c1468f10d816760d846585b6dd8fa49116f4e40fe1c77", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/040/review.json": "7f68fd58d6c3121dce2e8f1933def0db11cf3d8880a74545a783c043e728a8f0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/040/prompt.txt": "3b667cb13b9ab21119aa90bde6dcf07ab0c716815f30ca55f455b3ee6b64853d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/040/evidence.json": "5cec180dcfcfd72f01eccbf29d7cfe82c18070f58ee2c814912e9bbd19375d51", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/040/cli-output.json": "18c7961b40dd71512caf44a9a4aec592018552903ba8927606de71f376a01a71", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/workspace/supplier-intake/data/suppliers-q1.json": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/workspace/supplier-intake/data/suppliers-q2.json": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/041/review.json": "771b7a9f811fc78d9dfec808c17b88b169cceef4cd66ae2ce005a42f76f4f13d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/041/prompt.txt": "a2c77ba6722eeb5de8daec4dac69a555ef42f8039f27c5b1012d65a42156e7fa", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/041/evidence.json": "2d6899130f2e207fbc682ff9064909ee48bebbe01e5eb6db220d830a71db1559", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/041/cli-output.json": "dc03153d05b864200e993cf0409053011844a498fd53619d54d1ab50ddb51dcb", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/workspace/regional-sales-rollup/out/north-summary.md": "d688c6a4e997b50a885927a5e689efd18bc76856b17ec0e42a8b2878f97d22e3", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/workspace/regional-sales-rollup/data/north.csv": "1d565713b89e9dc055719ef769bf6ba5bea1ff888d309aa34767a5231a0af9ba", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/workspace/regional-sales-rollup/data/south.csv": "e68e801e3fc4700a2f4e7625ae8db017447668c1b36b0d19858bc309ac0be370", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/042/review.json": "df2e9b8095a71e4612c47be294670c8e6e3c6336395b7d13adc4c74f42f023f0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/042/prompt.txt": "b1301213819175f0798ea1381907ad040366c58c08d048bd74c81ae6a64f5e26", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/042/evidence.json": "9ff3acc79566b4a3efee6476d1ee0284dcce05b436cf7715f835a1ff566a7994", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/042/cli-output.json": "b0b016cddde6e6e16f96d393a6ca6973aea83861aac304abbd540ac3b74b3a06", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/workspace/inventory-health-check/out/inventory-report.json": "c6f1ae25c1892003c66b44c030cf8085ef72e0d161c0f2aa8c9fd9bcfc57d87d", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/workspace/inventory-health-check/data/inventory.json": "2a6f44cb0bb8d193c2abf527264e828fbf882e8a744180a786e784fe0688b439", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/043/review.json": "e1328a88968aa20510f620bef41344cec82492b4a0042c74801c1743546ca8ae", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/043/prompt.txt": "5ef82a30a2855a1c550469c93039fb7d17b0d6659df23db877caebe4d708cda0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/043/evidence.json": "e98867561f7cc70000472bf19a0cbf707251b23d179589736f6175e23eb50343", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/043/cli-output.json": "e3949190e0710cdce390012dc3e5813bc61735f02917168dd3c61cc183545f46", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/workspace/store-health-sweep/out/harbor-health.md": "61af50c42d23729eb420d3c7796931bf793506554db16d61a4dc0e9cf9119c7c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/workspace/store-health-sweep/data/stores.json": "8bf728bad631198134c68817a9d08eaf08804c45fe27af05c1d8433c7e573e22", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/044/review.json": "85ccc1495feb9a8207182f851b89db10d724969b16047475874428c03c9f33ba", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/044/prompt.txt": "70dc32e04966bdf716442847eeb7deeede110850d66ffe501af42482330c5971", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/044/evidence.json": "b874e9a4bee1b19edf9f3dd183d0cc334eac197345a5cd34a0f1815ada355af6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/044/cli-output.json": "02a2b01fd4598abe6db4c71a23038d8ab2f5898ee42dac396a759c19449a7cba", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/workspace/tic-tac-toe-game/index.html": "40b6117623d35cd7f9ce649498c8eb1a3b5aa9854c7de81a58e4605da4e4658c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/045/review.json": "3374dc38293ebbe5454ebf627ad07570e3418629573a1262130400e75d269c61", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/045/prompt.txt": "d53cfb34208c625decc214b58754396426eecc278a9e85fa48cf1d593f19ce12", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/045/evidence.json": "01d556ed55171a63419b247617006cf3934c8c2cf9c76f73d26e4af85d1903fb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/045/cli-output.json": "d999615269d8a92658d68507713ed84b7403c5d41c53537cacbbf9a9854376d0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/workspace/pet-shop-site/index.html": "b27ead1fed4c9cae18dd67eb65f60c5162a9ebf79ca4633f9a1be90ff29804d7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/046/review.json": "7d43eb93edc4f0dd51a015694b20b37a436a9a963be628dadd3afdbeba2c2068", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/046/prompt.txt": "829949d8efe47612cbda1d2366410685d0d0ecf9fdb2068acc940bf92406356c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/046/evidence.json": "b3b0eaed975396e8c6af0db5902eda8bbf002f82e1f1350a62a7195bbc44fc68", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/046/cli-output.json": "3aaffddb53c4080f6de3845a75b6b1d9f12d4c3fb5cc6853ead9b30635ea5de3", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/workspace/tank-combat-arcade/index.html": "5b9b9f429bf032141c4fc6e7d918b61a2726528a5dbdfd7c264a73bf7f6b23f6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/047/review.json": "186f633e1a14b3b324571ff5fb53ca0e533d9535cb55ff51f0c4d67df392b081", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/047/prompt.txt": "c368579cccbe40318f66379f4603e352b07d2d08b98432cc082bafc01cba7b81", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/047/evidence.json": "c20fe6ed03b509e51eb70f8e6a803cc90a707fb9bd10c3af96bcba7bcaffc493", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/047/cli-output.json": "999920284da8a3099e60f6dde68fac72281b0482710abd9510335ce070d80239", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/workspace/user-store/src/types.ts": "00844023ad6b9c14cc8d3680b9e289d9eff42196060a2b0f54b004ec01fbddea", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/048/review.json": "c34baf74630ca2355eb06619b9bff95fc78455c2ee03618abcabe3b6f2c115f9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/048/prompt.txt": "c007047017924e152c5330c2ee10b78d68eaa17cee66d3a4dda5175e57a607a3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/048/evidence.json": "63fd8cda405edc919e64e5c5305e9b1ad8c4bacfa6256d23707b6bebfd75f1c5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/048/cli-output.json": "d4f7f5d36211a18085572b1c7b2566ba54e0b64d67862799bfb9fcc59e5f9e53", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/workspace/machine-kata/src/machine.ts": "7d1a41e50124eab8690e29235d7f3f5a39fa830ee1c1c9281a3547f33150953e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/049/review.json": "97b2d76c3bf27411a41c2246984b6ba342a8f745bb9bd2be39eb07de8fa64fc0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/049/prompt.txt": "b766f1463f15108965891df6b51ad91997af81f94fc72e742da3aaecf69f1ade", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/049/evidence.json": "40a55c50ba43c95a96cf2c9cebf65aecf56b6e52b5b6a952857ad77728d3e435", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/049/cli-output.json": "3833bb915dd5a73ab4d04953ee25aef45d1890d1d7ae7f7a78ec8b4085070533", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/workspace/pagination-service/lib/paginate.mjs": "a33f753fa675cbf56b3c9066e16768fc555810fd79aa17af79b2d13a1b043c34", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/050/review.json": "e3dd39ed0686dfed541b6783496f470c3c5145f191d8abbdd2c84840f7b52109", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/050/prompt.txt": "54fb5989362d42fe3caaafdb7225e2a23a2a37a6dd8a6c35a0b51850e2a2a385", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/050/evidence.json": "e6639124acba222ceae2563cd01b5796b9c375f418c96356f961487e18ef47b1", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/050/cli-output.json": "eb0706eb7f5c4865bf3e1740d9249281ba6226c8e7d43c7744b99f9bdfe1c362", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/workspace/customer-data-cleanup/out/customers.json": "11bcc09a93017c22a2bc704f81585d33bc222c2aae6e19b48b52f98ec0342f77", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/workspace/customer-data-cleanup/data/raw/customers_a.csv": "bc92617373571bdf0bef1c29ee05aad44a7a68274c0f7334b1ebe8aa5eedfe54", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/workspace/customer-data-cleanup/data/raw/customers_b.csv": "ff167222077dd5a971ee88d6ab656d98b2b2051e84ceb223bcc893aab9186b09", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/workspace/customer-data-cleanup/data/raw/legacy_export.csv": "90e6abcfcf404de96b3ac6385e3ddc3265125552377f3c3428d5636d310f9afa", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/051/review.json": "607272f4b816b2804f88994b76ea0c99ea4df76149ec6c34cff7fa8662351700", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/051/prompt.txt": "09e3a28d8b59268ba8fc39797eb41d4315e74217f51cf6db1f538b1c38861b18", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/051/evidence.json": "95554f8536bbc8f245665026ff3be84aaeac9307648f5c7981c9ad52f0b08ab0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/051/cli-output.json": "bfdd470af598bf7d059ef48ce3fe152ef0f325b29fdd80ca37c262c664e33cad", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/workspace/checkout-incident-postmortem/postmortem.md": "c667729420b47878078574e88bcf84ac16dadf728a74c3087dfe9dc3fb23c7f1", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/052/review.json": "9594d415efa5874471721973e4d00a19c168dbf07cd424cac3a4b02d1af9076e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/052/prompt.txt": "d4415f3e245a33f1890cd576d091f60ad947074d9b9d18de50b795d70384bfdb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/052/evidence.json": "8d043ca5949a408cd7cfed5fd24d18d5241972e114faeaaeadcec0d630a15515", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/052/cli-output.json": "f35c321b170817538998546a9824e3b0801778f8478b73490cfa1a5b5bd0d85f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/workspace/quarterly-maintenance-window/runlog.md": "1655b285ef26726484acb4436918fd5224d6e5c2fe0627a6199023173e03d306", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/053/review.json": "27b050de6c7c95e09534a5673ea2760f43a5f07513b417dbc9c25c3da16f6ca2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/053/prompt.txt": "3b9898311cba07f1600d3e11add2c1882a348b126afbdad0390bc12f93449f19", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/053/evidence.json": "2a260f406bcc539f3d41724c79d024085e7c7e4b7f869a70808beedd783afefe", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/053/cli-output.json": "1417797b32638fc69faf5003b56d7820340dd716cc800bc785a055ee4c338ca6", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/workspace/harbourview-office-relocation/plan.md": "f25343bbb6419a74e0179f63807556b816e8e27efdfdcc81d1b144b80df894f9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/workspace/harbourview-office-relocation/brief.md": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/054/review.json": "fa855829d186834b0131a6633c6d30c4289f14dfc0af1e4aafcd763ab87f79f5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/054/prompt.txt": "ed6fbf42b9a4564ccb85fde88671cb4abc0b59153cdf4af09f4856dc9c109b12", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/054/evidence.json": "284ef9d7ce3d320469a65efa0f2fce79d02274856244988d9dd5ba2d986db871", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/054/cli-output.json": "4f637059c67c4b08edcb87883dc82cedac640e81a2eb9330c32a4f4e098a33ec", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/workspace/skylark-launch-synthesis/synthesis.md": "2046324a06760038561c6907c6a13b759ba2d0bc57749f5baa756b2b7f0d8df3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/055/review.json": "d0690004a7e23c91387632f96cddd6ffc15e4b2c5716d433c588a6e84dbb1d6d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/055/prompt.txt": "ddcc5275f7caf65beb2166854da2fbdae6e7972f9ce5f7a396e9edbbe3a17e3e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/055/evidence.json": "97d100fa6c9cd8f2e209e88acf4a20fe0dd1e3605a6f57524cb36c4d877e63f6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/055/cli-output.json": "f4e807523482a08078c3c051001861afda6bcbcbc79f56ab58ae1603e12258b0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/workspace/driftwater-outage-notice/customer-notice.md": "3e12989b54687213427048080c2158da75b6e1510fa6350ac6484a1fce22a613", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/workspace/driftwater-outage-notice/facts/incident-brief.md": "a65cd16d09c63d27126363a03f1bc664d26e4c6fc58eae192477d98ac3d02251", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/workspace/driftwater-outage-notice/facts/legal-requirements.md": "f15d48f9a7a37a2e3d855566ffd9cbdb9f0633a9f8c7f0756816b40a15815920", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/056/review.json": "ca8291f5f480e65607c44504390115a2692ef39e71c0bcdb395bd9f6903e7669", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/056/prompt.txt": "53a3ccd8bbe2d693712cd87c57ad9ba0ee36f4b6f2e178051386eb9f423efe81", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/056/evidence.json": "4bc785569616e13b697c0767f71f23944511a6878267844c84e23ba28b7c358c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/056/cli-output.json": "bcf63629551ec130fe0028eb03324047eed9fa59a1d6f605fffdbb569d522010", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/workspace/conference-registration-intake/records/attendees.csv": "e86246bb7840cef1091999c3e5e6f7da1e66d6e922e8ebd218c65d42d66e01af", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/057/review.json": "3ecf033daa1e89a40ccf4ddc2aac0e504cecbc8fc34bfff621e598a402b6cf12", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/057/prompt.txt": "b34b38647e112f34403464b54ed05db76dc6e3075525cca18d600ce0ed1daaf7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/057/evidence.json": "f7494cc8f6fada6bd56cd26163fb89980d948030cfd5903f00da5f9c8837cccc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/057/cli-output.json": "9c32f07295ea129289d392b448e1ca578499dbe40d6c66a5844e6785562558cb", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/workspace/harbourview-office-relocation/plan.md": "c5a52db34aef596fae76c4365222515a7ee030374f94825ae8f5a468e98fc5c4", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/workspace/harbourview-office-relocation/brief.md": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/058/review.json": "49c697d2a4955e4b898f67e70b75e3b029b2e2d3ed3941b443140daf950db532", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/058/prompt.txt": "fac01b9581b2288fe1987f760cb30f0584a73ac72a44cebcfe77062d6dbcb267", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/058/evidence.json": "105b2bec131fb33c01ae57b41e77271919a4ebe058abde1cd9abc69ac17ee1d2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/058/cli-output.json": "ac2a80ebbf9778e369d610261a0e87687bed2a9f2e859e56772c9f68ae0f452a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/meeting-brief.md": "ec12314025683a01f310f329639b32633adb2693d239ac766fcdeb5321eead59", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/meeting/old-agenda.md": "8a895418b408627cf5fd7367a45341bedc5fc29515c61dac05de3b47f6fdde38", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/meeting/roster.md": "8f77db20d7700521aecdf6cddcbbe807ebf99bae5c2043a558fd5f96831cad69", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/workspace/orchid-launch-follow-up/meeting/transcript.md": "7a4039bda8b643bfdfb3f8d1208a5c7835fd3b13f280867db9b6915992e0800a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/059/review.json": "9faf6e71210175151429465d421ab9dcc117f096c0f29199990c0348ae134f75", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/059/prompt.txt": "7fd3cd99fcfe2a9dd73bb755432b3948929aa3572099a2efb3af440657eaf911", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/059/evidence.json": "9b402432c211bc467fd3fec49f32c3ed857d5aa90f7c0bd067a821d45028cf56", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/059/cli-output.json": "42d9dbd4d0b5ef48b3ce95332058fdd150174fe05f3c30df72f4160b9951a576", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md": "ebb486d759756e80e34c83e237d250a191c18430d4104ca9b97cd278f6ebb0e8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/workspace/spreadsheet-financial-model-workflow-eval/source/brief.md": "9b7893b96d2e07d87b152dcdef1c8e16395615b8bc827a34cc10dd0706ab0bda", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/workspace/spreadsheet-financial-model-workflow-eval/source/craftbook-eval-brief.md": "6cebbc3d1462d96e4ee669842d5d1d5fcc2c9e0c95bf6e723aefa688b297cbdd", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/workspace/spreadsheet-financial-model-workflow-eval/source/records.csv": "430befe636c28a4b2751e842a79773440af0bd7a5f1b742ea177c6ef204d74c6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/060/review.json": "050bfff148243fb5c581e0da6f857f539258cd204182ecf063ff0616c6ede2bf", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/060/prompt.txt": "70720e7e0845d7b2ebf93ac32bd6d41dd9f2c6a91b11563c2e85bbb0d839eb7d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/060/evidence.json": "774eabb46e716ca081d3cfeb35fcfdfed109a7698ec242bbb5880d5d04dc8272", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/060/cli-output.json": "435e18af0563a6c8ea2b0cd638e0707026f417855a58c38021653a28f9f49116", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/workspace/skylark-launch-synthesis/synthesis.md": "0261efd8d80b109d110f652c0969a0c9b5dfd8bf16d2412548291379c0c9d456", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/061/review.json": "5f0f84863cfa7510135273114a9bc6e383dec23f982cdf34ba98ac74073430c9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/061/prompt.txt": "194e0189ba5b2ef36711c9d946caee0bb7d244ef2baa1623ed795266991c2f75", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/061/evidence.json": "e9d2ef77aa030a41e4e53c648cef02ef638965b10142df12b9c8998d875d82f8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/061/cli-output.json": "8d486852ebe9581b3c0d1270256b240aafa7b8b4138525bb15b9ca1cf3fa6a84", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/workspace/brand-theme-alignment/theme-report.md": "20bb01a7cdb3acbbce4e4a8044a36fbd21397a3863b1bbb8eef8691fe27fa1ce", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/workspace/brand-theme-alignment/source/quarterly-review.md": "70a119718fdfd89aba5d0dd2bdc2e7df8fb45f749afc71d56bb5980e1fb030a8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/062/review.json": "fa9e743ee0acccc4f02835a69bd1e4bdf62bafc33c871a49bf884812b72f72e1", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/062/prompt.txt": "bfce7a39e1e7829294fc5e6da51ff615daa80e909f1fecc8c509cf169461f994", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/062/evidence.json": "a1ba96b29484521e704772c5d71f7900d2ce8d65e2aff66a726b9d277248bdfa", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/062/cli-output.json": "c800127768105a30960cbdc33ad3ae2e5eebe987eee6dea5a2c1ffc6321ffb7c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md": "3f20cd4621a889fc3f154209a1f6bb1e3cee8ed4f6e875f829c5e90e7d94e784", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/workspace/powerpoint-grounding-eval/source/brief.md": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/sources.md": "dd8668daaf43cd2a6627bfde6c933aba98460aba3e76a259f142cd73544856ac", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/outline.md": "b3429eb36add706d9d49856e586eb4a786035aeba65a4ec5599f7720f0327afa", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/artifacts/powerpoint-grounding-eval/tasks/1/review.md": "3ade48915d3cededc76a5b50cf9628dbff8f8b79d81ced07c1fd615fd7e6b4ad", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/063/review.json": "6cb9af00b7a99b92d238dda7a27addc395c18f1d9cceba2b647576100a2b48f0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/063/prompt.txt": "59dcb2c869014ebdbac3d502b60833fe0440d90e8612bff9194aa6758c9f9971", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/063/evidence.json": "814ee012410acf7e2b6b0a4b25608658ffbae2dcc9a66eb03bcd7b8662a4de38", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/063/cli-output.json": "99e6720d122e99b4a38d25e1310be0c953e9e9c7d22fddfc5c15a830b627aebc", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/workspace/analytical-engine-research-brief/research-brief.md": "69f24b180c7892fa3d845fe45f8d2161070f05f3ae09a321204b1d0bc7b270e0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/workspace/analytical-engine-research-brief/mocks/services.md": "7809b583d708299ab83a2a997331fc3f0718e96d399f0c41a4dba1c6f91315b2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/064/review.json": "cb8bd75e14cf64787d33657a41ff7878c04a2038ec4a889b0d198c3fcc7c282a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/064/prompt.txt": "04cec19bf85d21f202797b2b6e462f423a48f8958b438a237ac31be03cfdb78b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/064/evidence.json": "4c07fe94b6039e843ff5d2551577b6418d1c23df28164f8b851557ae4615ffcc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/064/cli-output.json": "70fcad72b30073bb726cc867e1c92e67a41d2f013793ccda56bd15403de54603", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md": "bcc737b0c7a3abec6f21d575b1dbf19172e829593f91e069f7a96f4931d13af7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md": "bcc737b0c7a3abec6f21d575b1dbf19172e829593f91e069f7a96f4931d13af7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/scope.md": "f2d446853ff73f964b30100b4623420d9f5dc01c40b7681403fd67000f51c16d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/plan.md": "dbbfb68291423173f120007c968015be1fd80398891ab89078e3643078b960e2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/artifacts/calendar-desk-week-plan-eval/tasks/eval/verdict.md": "ed02e41042b99629e59f16ecc43214f877c7624a7b2437a938403599a6dd9c6c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/065/review.json": "feb0fdd640112ad5fded3ef66b8ec73955d6fa8fbc10a86cea882fb59ce0030f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/065/prompt.txt": "f9fac463d512f4fd7af64c2ed51d1947813cfa52ef8b0f8cbf9a2f181b6f91d3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/065/evidence.json": "0f1b65b4afa7cbbebf35b93e596e184f8c68ac1cc8740676275bbb59ffee036d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/065/cli-output.json": "32346ef422671c77c13f51a5b5463a1194f6d568c7ee8bd4c2c1940acb48f031", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/readout.md": "db45f2295dfd993bdf5970a8f551ebee3ffa6500f77e918771ed5eea18dfa210", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/data/stats.json": "4beb58b4e2879598b5a8ef6f441ff98d3be4a5e5936c757bc45b3c342c92937c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/data/checkout-test.csv": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/source/experiment-brief.md": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/workspace/a-b-test-readout-eval/scripts/ab-stats.js": "c777b8ac319357114d714d63a3aed1d5943e442ef8a7b965203527761e42db08", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/hypothesis-scope.md": "3a33bf1af55bfbe393eaaa35c6d3b9ddc3bc51abca19498512ef3152a22af1e8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/artifacts/a-b-test-readout-eval/tasks/eval/verdict.md": "0baab1766256270325d4ce46b3ded0ff7125088522c01eb40ad6df32cbbaa333", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/066/review.json": "39a202105f4240b7adc4861a357cedbb59d33cca909161a9adb266d53732e17a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/066/prompt.txt": "9bc18621a477f47a33c021ae378036d0a53657bfee6d16a1ebc6400de00ddee4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/066/evidence.json": "c8e6564161dc54522f8ed13259f73cdf4fa7ea2fd062eed394247575bae59180", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/066/cli-output.json": "6139b08b501f017e372a121b0a67129060e3fad140a7184ba0bc2b61830628ff", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/workspace/annotated-bibliography-eval/bibliography.md": "5d859afc3c4cf138476a201dcc25808e123acedbc525dc37fb60c2134bf31dd4", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/workspace/annotated-bibliography-eval/source/source-cards.md": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/scope.md": "64af78f5ab97e1434ead970e0395182896a9aea072f86acda070e0e1597148ec", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/sources.md": "27c12b61df547982f4b7f24f23a861f0f7b4f1736d7639fc4bae748ac44c541b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/artifacts/annotated-bibliography-eval/tasks/1/verdict.md": "42c0106e9c4085dd926746b433c4f0c351cc5fd7acc77b89635829af3c248ee4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/067/review.json": "de76ccbe7b289424f7809ea1a9ce0b60fe86f149f1e539f51ba25494c1ea0a16", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/067/prompt.txt": "9ac2e342e26609d01f6dadb0d9f944123e22a0ad4c88d29a1fac128f5c144a25", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/067/evidence.json": "5366c9b5e302ddc3fbce24c0d5c010f8cbfba24f8fdee79adb89d4d00a17c0a6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/067/cli-output.json": "def902e7df86d72f99e23c0d9827b0d1882c18a55f08863fe058ab9f766096b8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/workspace/word-document-from-content-eval/documents/eval/report.md": "a5b472f4a0c1e1860d5ee3ce070b57063773b94901a14b7178aee84f11c482a1", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/workspace/word-document-from-content-eval/source/brief.md": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/scope.md": "43b5fd7c883e3a46b72239d6ca6609d10e49b48653feadc459d76583ab9dea69", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/sources.md": "f6c9cf6e20372f26fe79a7f5549c0048c21e31fec9b8a5192e295ceb24a6409b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/artifacts/word-document-from-content-eval/tasks/eval/report_files/report.md": "2211fbd1926ab266e10b16376af3cc046b9e93e3d4794af9b04a75edca2f856b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/068/review.json": "bac61b72713db5c85350c7e10cfc9bf2dca85c3492571053252cfde48599c453", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/068/prompt.txt": "491583ba9d88d395b1e92a1c3c0fccd4852ea9a5c6964eab78f02c087356fce2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/068/evidence.json": "cead075fae2e203870206711722ff69902848e69d0a3cd7f94f8d89fa6e38250", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/068/cli-output.json": "ef56fa1b9021b752bb3d5b078d4efb149f16c51d030f7a403f189bfb15e23168", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/spatial/Geohash.ts": "cdca05d472bdd3b534e30691244fbdefb8501a4202b8049bac43530f42397643", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/imageEdit/persistence.ts": "15d7ab106f1e475991d992ec252800399a9d5539704ec1e54caaf7c7ab48b3bd", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/packages/core/src/spatial/Haversine.ts": "6ba31700e271f0f3a33aff0339cb30e902bb890c78d74bb9c47f2e4b06eac635", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/workspace/squisq-core/bug_report.md": "d44d282baa62957db8f6d72411f89a68d8e5fbd11ec4fa1721a87f779def6a2b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/069/review.json": "ab7ad1d4779b5fe8e3965b6e4e1d3aa1eb11f5a1b3d168a304bf693bcc2a7661", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/069/prompt.txt": "d5b38c926ffea4375ac4e1042bc9e6de370f15252746689836ca2fbc569bcaf3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/069/evidence.json": "90ea23b81b980cb8ed0e33456f57b7601a1eabc05af20bc0d1f7b6908a3ba268", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/069/cli-output.json": "f67a1a45d154fe111ef1dd6303db3f0f9007da6beebfa955f6af25334aa4988a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/070/review.json": "49fda8688fcfb2ce7e763cad6dc4cd5dff2aa9fc4cddddb104fb22b4ba974ac0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/070/prompt.txt": "127e8d9326e6a325cd641dff9fe52502e28f3748eef6972ef9147f82314d17bc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/070/evidence.json": "087cab241d6d161d0e1ee548cc6211675b0ff601339ca7737d92277a31e533d6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/070/cli-output.json": "4739761b1542e0ac3e017e2f4b9db27b9e63bf0d82a85ea37f01468c2690b486", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/types.ts": "90aef81b66c3f49e583a00a53971b4ad0547a74523010eb128fcaa94ba7ceb23", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/producer.ts": "dc3d0a72617621a1d3ea340669fe36734ef18e6758c1b6793a693355e157e00e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/workspace/event-pipeline/src/consumer.ts": "c63ebe35d14a9995ca104c220af08cfbfdfb9de1d6a97217f731ea9836d008bc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/071/review.json": "854545cd52b3e04ac5b9057ddf888f07bd0a836760f92b832cb33df050425db6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/071/prompt.txt": "b4aa71422a4c8b21d7cfe552565c7e2160bf094dfb5206deb49b01f7698fec66", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/071/evidence.json": "12a445cb84c8f5c4540ada29d128ea035f4e9759936a2f47f057411b79e2274e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/071/cli-output.json": "7d42063b5abe28da520f2634a439c8563fc7bb2024fee59cb36227d5d39207cc", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/artifacts/code-review-workflow-eval/reviews/rev-eval-1/report.md": "d1bc463d08f5b54d8214c4af82ca95e5f2e2aef9e10aa4cb0659f040c8f52925", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/docs/review-conventions.md": "c5070f220c00da5352d6129656e4f86fa7de4d70ea24dc7fcde871d59926b62b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/src/payment.js": "9973eedf102109a6708c3be05e9914611ea85bc1baf6310ea385a891516ecbee", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/workspace/code-review-workflow-eval/source/craftbook-eval-brief.md": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/072/review.json": "c06224346e1a14afc94552f9586553d4638bb339138b11b50fe1f72a64c7473a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/072/prompt.txt": "7b82466469420715d4581609377eeda15172b59c669f4f99c7750de33add749d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/072/evidence.json": "46b85a5a2fe754b87628574ad03add27a4a28770d52e4f3bdea5853c24ba6071", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/072/cli-output.json": "801b3e986cec8d10a8c6445bc5a9c775e4e0712ca456b2351f8c3b5d0ab9b3b9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/api-review.md": "e8713bf3d370d92787bbd1a4b5969e00312765d8cdcb443a49fcfce18bcbc746", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md": "f81e5302bf5771d904723d1a04f4baf35e07958db1627b9e68287aa7eafd9498", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/review.md": "56b3778006a3a9e5f1446e86fe709aef695c5068248351e2d4d7f17139a33b61", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/artifacts/api-contract-review-workflow-eval/tasks/eval/verdict.md": "89398172ee9176bbdbfc473ac534937db93c6a699a12b57365b96b24a0ab3982", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/openapi.yaml": "b8066a1e5dcbedcf09f879f6e9c117465296068a7bcaac1b49e106dc6e6ed36a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/src/routes.ts": "711c645ed2041fcd74e725447dd6e5cdd06d3d2663092f21ef6fa8c2b57b1453", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/src/session-routes.ts": "dd8ca7e7f97f7869a1f28b32521ff82c2d254e21e91e87a9355cb56965c577f8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/docs/api-contract.md": "e785f70ecb9595576090dbe57b5770356b373aa87b4318c5e6849612172253f1", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/073/review.json": "2eb4184a57176828bcdaf3e6bdca7613560956753770c01ce429e034cfb1cdc8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/073/prompt.txt": "07e1720e8da315205b1709a066c7090989c1a4cd5dc56e3c26b2ddd5cc2ef5a5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/073/evidence.json": "05b12393970dd6f183d6a7a1df3b37214462fb4169edfa1a510734ec540ffc2a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/073/cli-output.json": "a95387faa0fa23bc0668735741143360c538bed99300971a56c70af4ceef1400", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/security-review/REPORT.md": "4297875621ea5896c67b58dfede56cb5747ec79c467efdff84b7bd80ebe47d94", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/security-review/findings.json": "7d99283f155060a12be807b165ef9366e3249d31c63f4674f6266932ebac56b4", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/admin.ts": "5052e98121fa01251d8fa8b3897505a5611938de2e59fffe6a98ded0254ed5c7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/profile.ts": "5a8c648929d5a44f6a7d36728c1caf76a35a484b6164404cf880b5d54341aa29", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/src/reports.ts": "db992e8fd835de35757e54801468aebc219c3150354aa629adf9a1c301072790", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/docs/security.md": "f0b35d80c2d43268c38e6b2c4a8639d01cefaa59516663be1e244473cfebda5d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/workspace/security-maintenance-eval/tests/security-oracle.mjs": "e1736cd4940bfe55bac7900c7175d3202343ceec38f6d9a7da17e5e398dba6c7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/074/review.json": "729d7043a7cc7b651ee25f165cd6cd7bdd1e53fa565983326c6a6f5a539def09", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/074/prompt.txt": "05d51ebeba3f4837175fa7316cbf70f40f6836031e444a87fc0983e072619b9b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/074/evidence.json": "33e366e34f7d7f0d7ae33d57d47c1a28681f0be8f455dfee9b02eecd4a248d40", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/074/cli-output.json": "d2b64fba4bef633640a7d57ab058176c323ddcfe0d9f53be364e669e0e7978b5", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/workspace/large-pr-review-eval/pr-review.md": "a93d7c9bd0c9adf291b11310cbb11198955eb46a3a0558e0e985260fe708cb51", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/workspace/large-pr-review-eval/pr-review-coverage.json": "d67e7e39a91d561d7cc069143cff8f1f6bb91a7a21a5dc1c09af03c617157b3d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/075/review.json": "a1139342832f884faaa0a89129b2c385ed8d140f464268fa8bf74072ec5b06e2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/075/prompt.txt": "859033c16cfb1bbd7776e18eaf273e691da80dbab5bd579042878fa97de0bf13", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/075/evidence.json": "7a04b4043aa277e38ad2ca5b5b2e17accb0eb28c2a33005ca86a8769ac1ad8c7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/075/cli-output.json": "b8681d53298284450306f3cf617297d497de59557fea6698dd71661fe384b8a0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/verification.md": "ec138f2348265a86691f29c8a26e17a3bbbaf146bbd003c56aa6cbb1b34d2674", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/sites.md": "194616bcfd95e2d2047c141b4706d4debe31e03c6b98614f0dc081860e850558", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/change-notes.md": "27e1914ebea58444385d008157ca2769b9b66220f8d633492bf9f8ad9ac860ce", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/artifacts/profile-cards/tasks/1/review.md": "13b7813f1f0cbd6419cd56743b99c1592fcf686237a86eae32d2c2943c68fdeb", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/workspace/profile-cards/source/rename-brief.md": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/076/review.json": "7b06c7cd733afb698146c3b0d36a707fe3a2e58fd35f086e9a8d75c867ab8549", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/076/prompt.txt": "d97c75ee463e5a2776bf7e3f9a60e253761b7382ad06767476501f995f9b55d7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/076/evidence.json": "13294fbd85ab6302fe5ad3731a5ad80be63908ff4146c1fec24b5489fcfcf300", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/076/cli-output.json": "8caa751b2aa1f0d547294b4f968bffc88947773daab9ad7ae1bcddb2579dc73d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/artifacts/crewlog/tasks/1/change-notes.md": "df69bb8690f5c9d01a1b3ee0981558d51b338e5a3bdfae352cbf088ed2040105", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/baseline.md": "59b4b17b08baf8ed76ef668eb72d92fe4408ff25eaf07ea5bff077970417d60b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/plan.md": "047bfbff81a8664c5e0835f990c8ea36a57b1fc3341f7bf0fe4bfdaa4099f0ee", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/change-notes.md": "df69bb8690f5c9d01a1b3ee0981558d51b338e5a3bdfae352cbf088ed2040105", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/verification.md": "6eb12dc8dac471f46554d4a6bcd38b9bb05ebfa8bf5a4d0d0b4cea1c264abd67", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/advisory-progress-2026-11-06T0406/review.md": "b33dec6fa53b1260206bc70f98832ca188ec93bdbb08ffd03040b57fca28b0a5", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/workspace/crewlog/source/refactor-brief.md": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/077/review.json": "da042a595f19fc4a8aa268a6be5f6edce7ff1da40d6125be266913f19a6a96a9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/077/prompt.txt": "3ab8eb18ad6363e3ac6c9f67eceaf9c454445a68c001cf7542e307acee676536", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/077/evidence.json": "ed2ee69ad44f99e527eecf4893a41b122dbcb8d53b7a4b40dba60618b7e08c94", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/077/cli-output.json": "65f303c7865299784867281b6d5416853a55157075825a430e4f63e8833ae1b9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/fix-notes.md": "ce4e8e329b8e5cf6bce92be568c6f7db43162f95597dc94823ca8c9a0bf79717", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/repro.md": "137d52182fb37982d2572fd6f280a45400f40260b2ecf84210ead48c17539d0f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/diagnosis.md": "bd5d7b0d6bb5126125fd7c8c9acf360e7b9e2fb9cdee8e88c1e5ce6833395f18", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/validation.md": "35fce783703120e86ef729f1d7a265a494a255d9d7b2c8ca524ea06fff35eaa8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/artifacts/orderly-pricing/tasks/1/review.md": "7a1f49866f59647d139c7ddc81f07683d3a186faf2cb92f186016e879d76efd2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/workspace/orderly-pricing/source/bug-report.md": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/078/review.json": "2000d38370d06d273aa675518bf84be826c9ce757ebe2150f61d5286ece08529", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/078/prompt.txt": "76df4f910632107e5762f93ae7182f34d27e2c7cda3c3378727e8f725fe9846a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/078/evidence.json": "addec914e9a6117ce497e225f8e88c346e86e3dad6049b4eefc23358009af0cc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/078/cli-output.json": "db8a5cbd6959fb29b31fb8673b7d176c5dd499d8a81faf7f2cc19983d1dd8b7d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/079/review.json": "eaff9a280ecb3156ddf9d87b8c964d73e190771647358dc97523e5f476ff979b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/079/prompt.txt": "5163c1fb5009d9c76556ff5de9b48a4715479e5a80cc4989d4e9da9f104a9fe2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/079/evidence.json": "d85a9b0b038250540f782036aca801afce1768f1610d170a2fe5e30c29d1e810", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/079/cli-output.json": "9f796f32a549de941130792e9d1c86a10c3969660a26dbea7ce6d24f234f12c9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/workspace/quarterly-brief/out/summary.md": "92ee2217ecbc95d317ba6ab358968b47e351c1187a69563183a2a6bfba690e84", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/workspace/quarterly-brief/data/notes.txt": "949c911a4bbdf1083c68bbe94d539edda30128a353f4028a07a65bb717752a47", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/080/review.json": "31d5303af64fbdbea615c4c8dd27e3bba0df13806686c151938ece6fe9166f49", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/080/prompt.txt": "2fbedde0cb94cf9fe242ed85020b31a0dfaacb755f34b9e2c5008fbee068e0cd", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/080/evidence.json": "98e17ae101227a14fcc2618a48e69a3a917cc613e93ee12b955494187a9d2265", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/080/cli-output.json": "b2c1a612f945f8e4a8081fba84b01a43318e45773ff66be2327635fcc9637051", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/report.md": "a2ff64638c96f508bb7387010572cdd178cd63287a738bd3160398c100a5bb01", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-042.html": "1f2adbda22af6b760e3e9a6ddfa56fc8ef0e423ae88a5b281e0a7041b42d8acc", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-043.html": "733142de47893d7db22af717a1c3bf1520965829851d5051cc287b8d2bb015c0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/workspace/fieldnote-studio-office/invoices/2026-044.html": "052d544d5692fdc90d46183002c18162d1b829cf38de9c131223932af396a4bc", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/scope.md": "e85962d2882ec50bb2d22df2ec32d3dbdfde4c2ff9cfd756495b131b5dee6a5d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/billables.json": "017d6f067b37b6c7093ef0910aa775113001675565ac46ca9c3f48aeec5adab7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/draft.md": "b75853211d8b41d1369040e0855f7f39ed32776c24c71ec3cc256a2143976205", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/artifacts/fieldnote-studio-office/tasks/1/verdict.md": "15ccee52315a88eb479a693a8f213b251d89b38ea18c6950e5039a66e6c3a74b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/081/review.json": "353408a9ef2db7cbd807760dc7411905562e5f5c0144cc046002f0baf4c36215", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/081/prompt.txt": "c36ed1b84d7dd2e8a43e5f995c15273f64b4bd67f25ea8579abb2ca7a1fdd306", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/081/evidence.json": "03d00e403eb3943a6c8f2d4e2bd602bb076159c5ba08a7a1de81a8a8c663fa8d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/081/cli-output.json": "4039c67cfbaed3c1142155662c617608e1c04785630a304eb4f114f8e6a89b4e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/out/report.md": "9c74025ac2a91380e3402f4e2ebfcc4ecaf17b35028b94f747a43cfd3998aa15", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/notes/anomalies.md": "5e8553e63a4ee5e618f483491f4ea9e585cdbeded3b336271c4b88ec24ed8861", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/out/orders.json": "48b428197919ee82d35eacc53deebe9545fb28e4e749d23184fabefea409d788", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/workspace/order-intake-cleanup/data/orders.csv": "c88785ed78c4d8b14484d8d0f04516c9c93c99045b2e90b60b19f96ee17d7bb2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/082/review.json": "7923ee36bdcd11a309a64013ddf284cb1e4bf622424c1b3871bfc81e26be55c3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/082/prompt.txt": "5434f35912179b3a3a6c4fd553fefabae3d433d50d62f15ff994aa3e7443c8c7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/082/evidence.json": "1201819b8eef400e3e68124e7d9bd5fd71d280a2b43ddadfb9b2f75ddc8fff1f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/082/cli-output.json": "9f82254a83c0ce558dd3a7be2444ee1b776a537e3283245426b5a4e84840a32c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/workspace/fieldstone-launch-comms/out/press-release.md": "ee786d73e71aab4a898401a62d147247cfc9b29f43e2c37570080be5ba1293dd", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/workspace/fieldstone-launch-comms/press-release.md": "1fbc941c162ae5a5f74c0d40d8c2afde3b9052916c62cc918d0f0cd9fa8c9dc4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/083/review.json": "eef5cc61c025f4e394bc134d8e24fa95832248d9b68b8e22dc430018009d3c12", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/083/prompt.txt": "b0924f790e9ba1fe2d936e40a350016db690b1b21a66cffa12ea13a5113568ce", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/083/evidence.json": "41ff2cfdbad6fe90bd3ed5215d863738db69eba32e407b4639b80df22aa26d1a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/083/cli-output.json": "fb0f4eebb1b69e73b364fd6d861dfa10bf115056999240be94e600a8b2079951", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q1-intake.md": "472605a072228a3bb79db0abc8ff74da67ea12d34a0478d3600aee4168affd4c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q1-clean.json": "5afd1ac820f6351024084c81a0821378c962dbc520a5a0c90b987645f229138a", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q2-intake.md": "07fa6968853146aa0b7eb518820b6d659091747e2c72bf9b4c4fd9c86ce26fdc", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/q2-clean.json": "bb337c953adb23147f6ed058000f57ac7dce3620b00aeef7aed22a04e169e20c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/scripts/derive.mjs": "de4be8a1a03802fd062777c7cfa254b28aca414b1c54669fd8f9ba0362ab28d8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/scripts/intake.mjs": "a1c3e9d812878ba494308147fea7bfb181df07e8941bb8bc253ba6027d46f08b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/index.html": "5958de641e0517918ffd4e1bc486eb7084a5854991f937fa7485d05e073602a0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/report.md": "dbc6d2d5916db8386b7845c4e0bdfa10e763688db3c813773a026c1b30659cfd", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/out/{quarter}-intake.md": "312c2f5b6f60f425edb28dc2b37808e5f0c47ef68ebd37fb5a33cd24af4da27f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/artifacts/supplier-intake/reports/supplier-intake-report.md": "3e58d1cbd4a15e7db1af2474948308ef4f7245d870edc8b68b8f3e22603b1e45", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/artifacts/supplier-intake/tasks/1/verdict.md": "d43f1ba868d0138992a3061dfcc0afb4456e7900b207c29384c5140e160d1a07", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/data/suppliers-q1.json": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/workspace/supplier-intake/data/suppliers-q2.json": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/084/review.json": "61af7ffbbe9bab66ccb1ee663d16d9e0a90eff4bc08f273a00ddc4be1adb3ee0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/084/prompt.txt": "dc53de25b6658c0463d60f50a4ebc340e0204d326f3901b573abfe75d0742d0f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/084/evidence.json": "4ce6e6f3c1224abc32f2096e50ac59c3143877868b358038c1f71d7f57bc33c6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/084/cli-output.json": "d2635d26fd5fd428561e6d5ad58537d1b7f016dc66fad694fa1bfabf0ffff756", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/out/north-summary.md": "c76d5f80a399dd88940764bc1c364f1f83b9f4873ed65d5e6d049976524cc4dd", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/artifacts/regional-sales-rollup/tasks/1/totals.md": "8ed22ef4e5cb63198bfa3ad34ab80152b4e29f8d0bba27bd4959e29a0b0d8918", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/out/south-summary.md": "1949e46ae064e49ace7860f4cf9ae2814119f64f2f1398b786a3550506ca531e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/artifacts/regional-sales-rollup/tasks/2/totals.md": "68973d21cc3360b8f3fdce3f0e75f16c94be2a10379edbf6c5bcdbc7ca893f51", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/data/north.csv": "1d565713b89e9dc055719ef769bf6ba5bea1ff888d309aa34767a5231a0af9ba", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/workspace/regional-sales-rollup/data/south.csv": "e68e801e3fc4700a2f4e7625ae8db017447668c1b36b0d19858bc309ac0be370", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/085/review.json": "7659218196e2618d3d8d1322681dac8af2e6e9d6137faae63c092fbd7285cfa5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/085/prompt.txt": "0ddc18591e102a582450701b60b0b0780269b2fb2edbe2d9ca780e754a3b64f6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/085/evidence.json": "3d9fd32345ce887be4c9ff3e2bc5725dc9ec08bfd8426ef7be3a19442f7123a9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/085/cli-output.json": "ee1a48b4558c439107f956476a6cb07d16c210501976747f6b93e9099cb1d192", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/out/inventory-report.json": "c5c121f07321c7967dfc8a3f1b4c057c2e247d4aee5cce31e537b521e3caca05", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/checkInventoryReport.mjs": "b4cdacb4ec2d1a72290eafa02b4586635175fc395d7a245a93439324d79aaf66", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/craftbooks/inventory-health-report.json": "50210a084f28f4617f24c3b2a389381adeb88169b9f58a3cb764f217114ef536", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/workspace/inventory-health-check/data/inventory.json": "2a6f44cb0bb8d193c2abf527264e828fbf882e8a744180a786e784fe0688b439", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/086/review.json": "417dd52a28b92d34d6f24f9aa643bf6d5cc4a022b796d4718ec1e6a7a8081b7a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/086/prompt.txt": "7ce6bed90d027222d47462f32f4c1a7d69d17df1b74025c00fe4750912e8cd44", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/086/evidence.json": "2530f28711e6e91a207315bc8da0bac0669ec6f58b1b39c13bed5530a67100fe", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/086/cli-output.json": "714311681396f6d53719cd6d3121a8cc3c5bc60d33730c1caa0741ed5affdee3", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/harbor-health.md": "d7426541b3e86cfcbd8c7299e359cc06f1bf27fe9e714558212767ec80d77c5c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/data/stores.json": "8bf728bad631198134c68817a9d08eaf08804c45fe27af05c1d8433c7e573e22", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/linden-health.md": "8ef973d93c6c903ceef4059a0d0b632a5c5c0622ca604bda6e03e9905d7e2a87", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/northside-health.md": "bfcc1c20556b9693d78755dae60f301a4450c62839a328e3c4478b8446f3f435", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/out/milo-health.md": "45bb8c573b603063fe068c20b57da13f49044995952cc9005ffb138fe8560ca2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/workspace/craftbook-store-health-sweep.md": "df900908211646571494432819ebec3f794c2aa78822a9555f14bf19ef4156b9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/worklist.md": "059540c626d17aa8c9f52dce2b865c5b10adde65e720a07997f0ffb252f08314", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/manifest.md": "dbff95f7ad5623fa4bc6b7e3082d83097c08b7c025b12f5f77a0ce34ccf865b8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/artifacts/tasks/1/verdict.md": "3735875664cfa9b09eda3339bb7e7b9aa160f05dc4dd3cc801f59aa64dcafa69", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/tasks/1/task.json": "3d929187bbc6b79fe9aec65fb4d2fb1c2bbb64af48c0b466cf8651eb7a7c6d6e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/manifest.json": "e9f4072dc1cd176e8261006c7f624b5114b5e93034a449aa0c4252db11a2c960", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/versions/1.0.0/manifest.json": "1ddd068a6da20d94c118e4975705ea4a322d2407b99a33e41ae7178a9c5b56e5", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/advisory-interruption-2026-11-09/craftbook-templates/st/store-health-sweep/versions/1.0.0/about.md": "6580bb27539016e63761ba25663f27655a036f003f8612ff15c051efd52539d6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/087/review.json": "589c41c7675f616c8f2757e84173f4a684a83c16e3e5873ceecab53d3af53ac5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/087/prompt.txt": "e73eb015692b5ab2d27a3c56569f1d57a08afc27b453ed54075dfaea2cf83392", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/087/evidence.json": "ebdb82d12b86e45c96370996f05b8bf791dc5aac1cf9e0d602eef449167bfc5f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/087/cli-output.json": "8c75ac632a3fe7887d3df2840b7e8959b9a61734100ced5ad9dc9bdd4dc89158", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/workspace/tic-tac-toe-game/index.html": "ba1e579a210cc799b723fdb598c0028d92b7b7c2e1898f2bac54379efb039468", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/088/review.json": "7bdba789626aeb07c3223b657265a745ffb6c8d3e5f37c2638b05e161d05d1df", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/088/prompt.txt": "7797a3a94468547bf9db38b78b2412d005dd8aeb690066fc06df5ba57fe6d551", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/088/evidence.json": "30a90d003a35eb6d6d4aaf93fb1dd8bccaea9d3dcc24afb8aebbc18ad7630723", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/088/cli-output.json": "53b5436c9d4c88e190c498d2721a589697a1503aef951f73dbd62f222db14e88", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/workspace/pet-shop-website/index.html": "0d7672bfd2a7dd492f3cfba0e100cab268704f42627f8749682a2b4d266605ff", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/089/review.json": "f0d7d558e0c3fda38a51318abf04ebc0a733bb27b9600e60c1b938698361d28b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/089/prompt.txt": "ff8b50aab66b83dcf90ce7bc29ebccc959ca8da99e76753dad0fc6e3a98f35b7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/089/evidence.json": "546e7458a1017e2d08b477b24878ef4253e7702ba500522a7f536bea4355e9d5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/089/cli-output.json": "240474f791878a490fad5316a4a7c889a4c549af1378ba0a5ef27fdbea6b4609", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/workspace/tank-combat-arcade/index.html": "1a65a406515adcda74fc10b0075669a872900b6d94dfc9d510008d4786cdb0cf", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/090/review.json": "6ce833f5d6f648238eb46ef56cfb114bc4b2f014a249714de965c3a90b859005", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/090/prompt.txt": "fc0cbf380a53e82ad7c023e591c5aa59a577a1220c19cf0a377479a881326696", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/090/evidence.json": "1ad8aa0424f799f93bd25fceafad69d581fe17484739117d5449999ea619fa08", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/090/cli-output.json": "6c91d44ae637e6b4a9efd11c950e1f31ab87d971a4b4ef880ea705512eaebab8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/types.ts": "82e1cb9c5cf8038253d7b9745933e9a7aff74ecfc918380eb303ca7b9880992a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/store.ts": "6c877c17a044a87df7ed7af1488eb1f879d841902fd183eba20260a8fe34c735", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/handlers.ts": "f151082616220d177a6e1bc1d525727b820376df24996db8e901924a3d2b1b85", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/src/migrate.ts": "21d86a7450d0fee85195b4705ef4cd0ea1a751975a476e418d91ad73803c3793", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/tests/migrate.test.ts": "61dfe278d9d8c0fb5000f7f1d0bc91cc47b84ea476f53c1ffe4faf0aec075b7c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/workspace/user-store/MIGRATION.md": "ef9269029a603bcd84a9add231ba2c97c8e5d3127ead63815d29c780038fd74a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/091/review.json": "b98d933d590bbd84ddfb9536e00325537347e485389400e66fb18d7bb151df5d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/091/prompt.txt": "a8797968a687f1f10f5ed4a4b986b6d83ea7fb157d76ded2a46744e1d278eae4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/091/evidence.json": "a69bb9c2d981b2af5194e3f8ba26b8d9063f89f67eac6684e40990ff8ddc9c52", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/091/cli-output.json": "44783ec902ab2d47315edbc6c8552709c7bce6f7b0fb4833cdc39e0a2ef97ad2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/workspace/machine-kata/src/machine.ts": "e6c9a3e697039d21f5fccfd5bc15e5eb821018d44d3ffed265c83e27b6d73c1a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/092/review.json": "01c836daa9e4beb8dd76b61ca07fe17d510c04bc17a52435848477406dd2e843", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/092/prompt.txt": "c91fed434df632eb751a4d8ca3f6be968d63168bb006f9f84d59fc51bed30099", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/092/evidence.json": "ce0558deb5fddb7bf2c58cf225f102dccf6dfc8740803ce397ac0d113627af0d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/092/cli-output.json": "ce6dc38801cca2741d30c2c92bfb3d1c2b92ba59909a7578753b08501769a968", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/workspace/pagination-service/lib/paginate.mjs": "626a0f50583cc62bec0d3048b839f084b86cb983c10a071d829595cef942ee08", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/093/review.json": "acf76ff5b6623fa367b8bc2759ac4d2781a7792d8d0e9a1720f214d41b73890b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/093/prompt.txt": "4572781813a0fcbb38fbfb3870194c58fa78848c3853c334e61df9e1ccd218ca", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/093/evidence.json": "b540c7cb170d7d87f198a63167b19fea2055e6548439cb3cde1f32dae65a2fba", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/093/cli-output.json": "367fe5f6140601472a7862a99491353f7a476831da91566d9d5596ebb97f59df", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/out/customers.json": "94e094e171033476d9560d54b651ac8fe1537d56e13d1695f283714925ca930a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/tools/clean_customers.mjs": "170b3caf0c4657e29d81c83f83ef53059eb721bdef7005ad707b1ffa59f5f783", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/data/raw/customers_a.csv": "bc92617373571bdf0bef1c29ee05aad44a7a68274c0f7334b1ebe8aa5eedfe54", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/data/raw/customers_b.csv": "ff167222077dd5a971ee88d6ab656d98b2b2051e84ceb223bcc893aab9186b09", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/workspace/customer-data-cleanup/data/raw/legacy_export.csv": "90e6abcfcf404de96b3ac6385e3ddc3265125552377f3c3428d5636d310f9afa", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/094/review.json": "ab09647a3075d32f42a2f3c8a9e24a7e8e5dd0a5d2ef86562f09ae22a43d5833", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/094/prompt.txt": "b91145064ec2b91ff32f7158500200b77ce43234e8027f856281e089047c78cb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/094/evidence.json": "eb46555ae872253b90bd3a94b6107c974eb2637e9fc3e45f22f654fd5f7c537d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/094/cli-output.json": "8e456840694b1762839e8f491415a55364e66d87f9f0a8e2038cc22c397f11fa", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/workspace/checkout-incident-postmortem/postmortem.md": "54bcd0284f45bdb953d1bac09583d01e23f70c5d33b7526b0c5245992dd277c4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/095/review.json": "29626665aa40254f3756c3d736f9c6ba050f1bd599b19fa6c14bd2de317598db", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/095/prompt.txt": "d48cc07e7260aa34224c4d199bb1228020f7c0eaca3426a3b780f42eef59de89", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/095/evidence.json": "800a44ef79686754fb349a514376fe6c4bb5c7178bff359e2a03469c2a9401c9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/095/cli-output.json": "afdaa90df66dc3946d593b0a6c867ef19fcb6008a5005a714a5205ea93ec151e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/workspace/quarterly-maintenance-window/runlog.md": "fededa7a5328e494c002b75ab05a12d2b2857944680eee204de37c33c646e887", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/workspace/quarterly-maintenance-window/halt-report.md": "f12d9dddfb69f7439f8f33cd8c2216bb3fdc467a066267f69ab58ee9cab8ab3c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/096/review.json": "39d19d1472abb7071bd838afbc7635e9a687bcdd97f57bc5417e22c5f4e7d36d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/096/prompt.txt": "1de2253782ba5e0b7656c38af745cfad9d93278a2de6464cbc5ae01b5f89a68d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/096/evidence.json": "81739e20bbf9f17e0d5c331bc94ca1a343c713b484997424f28e896d49e5d11f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/096/cli-output.json": "c420f2a033b84608d7f607ca339595373e06855776ba2a8413e67e6407cb086b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/workspace/harbourview-office-relocation/plan.md": "a154234e5c5575aec6750db9f8a1a2c737b42681248dbc891ddd511b31f2f1c0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/workspace/harbourview-office-relocation/brief.md": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/097/review.json": "1f681d5ebafef75d3768a0f5b6fee0644032ae1ede3835d0585f615c451591bb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/097/prompt.txt": "8bd3b6848faf684c030cc8494d11d0a551de993174fbff329c9f176a94a836cc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/097/evidence.json": "fa86a5eccd8a014440a6d7718c45932ce84655ef1c504b2bb8be16427b305bd9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/097/cli-output.json": "a022dd84a9132b76061680a5248175b207f8c83fa8e3a207643d51a3e757334e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/workspace/skylark-launch-synthesis/synthesis.md": "a792881ac00230d2af739cbf48484198c2dd5d7570bcbc45bc74aae2dff63051", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/098/review.json": "71a1fe30f251fb3469966f26213bfe3adcfee89c371b3bdda4da02b17d2079fc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/098/prompt.txt": "ded5778d8118649233b6604418cf4746616521fb8751494db13b668b254db5f2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/098/evidence.json": "21e6093a6fb5d597625413920cc3042964cb4d428d047deaeebc0124081d57d6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/098/cli-output.json": "b42cf42aad893dd6402c5f3270fc47478765dcb4eeadd9bb0bb6b52d49d49df1", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/workspace/driftwater-outage-notice/customer-notice.md": "d35246fa00bbeb22fe0e7e20040bc252044a41ec4ce580428537223d1b660062", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/workspace/driftwater-outage-notice/facts/incident-brief.md": "a65cd16d09c63d27126363a03f1bc664d26e4c6fc58eae192477d98ac3d02251", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/workspace/driftwater-outage-notice/facts/legal-requirements.md": "f15d48f9a7a37a2e3d855566ffd9cbdb9f0633a9f8c7f0756816b40a15815920", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/099/review.json": "5ec703260ed97eda461c141f209306e82bc954f42b9a55b3365a126bf9e1d949", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/099/prompt.txt": "51b32c5c48e46bdf3ee903fc55742a723aff415348293708ece1e2205270f611", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/099/evidence.json": "0b74efa7a1f14c0dca1dd863cf137832ff38cc1258bc6c09083d65165e4c20b9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/099/cli-output.json": "e7d5e0a8e26cea1e57ce9ede5a9b23812122feb251835762abab5d375a1def2c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/workspace/conference-registration-intake/scripts/consolidate-registrations.js": "bcb3813d8b6feace059f0db8537ab105a4a5966abae7a673de3f0b7ff4aa03ee", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/100/review.json": "64e608ac95e8d4110acb8fa7e0f54b9130ca3dddeae6692527b5c564023863b0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/100/prompt.txt": "a618a0245b1162793edc1a401a5e449dc1b5f6e5934bdacdfef4cf07e6dfe0ee", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/100/evidence.json": "bea7f0f964f146d90b8ed03405829b09f28d79e55eca39a2380e1f16a7c72573", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/100/cli-output.json": "0c856737c4f83a00b87801d3563c8096e9c56416b09634364585c8d34df9c38d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/workspace/harbourview-office-relocation/plan.md": "da76c8bbf2d9585ae26910a979680dc08cbfa39099574bae531713488d121a29", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/workspace/harbourview-office-relocation/brief.md": "c477d71f7e43fdbca2ee8909ce78c886eba0c6cf3b431ff9358284c53031b34d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/101/review.json": "8fcc9a05270fe851df80ca3510ae32b9f0b68fae6a92117edba097857be1d1c4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/101/prompt.txt": "7aee9ecd7c72de354b4835fe3e00142a61b1f3ce7085784c230adb50121eb463", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/101/evidence.json": "7e71eac0d72536195efb7ba710876832f6c063cc3fc7faaddeaacf0019685f10", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/101/cli-output.json": "63326a084d667920028b6cd534fb78948c6327d585ce90ed0761414a45026115", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/meeting-brief.md": "9fb585f34d26808576410c40a738bad999f9eeefc09437739fb09b47475f677c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/action-items.csv": "2c081c4b560ff6b6fd8ef6f24f9d4f6031888754ec558fd15adea0802d6ca31c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/meeting/old-agenda.md": "8a895418b408627cf5fd7367a45341bedc5fc29515c61dac05de3b47f6fdde38", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/meeting/roster.md": "8f77db20d7700521aecdf6cddcbbe807ebf99bae5c2043a558fd5f96831cad69", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/workspace/orchid-launch-follow-up/meeting/transcript.md": "7a4039bda8b643bfdfb3f8d1208a5c7835fd3b13f280867db9b6915992e0800a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/102/review.json": "5a893ceb21ef92f5288e8a79c9bfec6545123154f5cece9efbb8a75fcb9d5083", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/102/prompt.txt": "4fe54a81b889e2c16dac95dc10285308eb7bb37255c3bde2f849877feb527540", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/102/evidence.json": "ab899d9622d1029c70c1695eb2e6316266b2ec763e3d53f1f78a81b6ce4e1e49", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/102/cli-output.json": "a4bb57c71e2b0ebdb5a561b3a894fa5a47d9e78d4e0265a9ca5c2c58795ef195", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/artifacts/spreadsheet-financial-model-workflow-eval/tasks/1/scope.md": "18638d25f76746720335eed029b4f6a1c215a3f6ea6a1f16d57f9d29773dc6ca", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/workspace/spreadsheet-financial-model-workflow-eval/index.html": "465c2c064f2d7298945ea39b8847befaabbf585dd25024a3ecd270b6925d31c0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/workspace/spreadsheet-financial-model-workflow-eval/source/brief.md": "9b7893b96d2e07d87b152dcdef1c8e16395615b8bc827a34cc10dd0706ab0bda", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/workspace/spreadsheet-financial-model-workflow-eval/source/craftbook-eval-brief.md": "6cebbc3d1462d96e4ee669842d5d1d5fcc2c9e0c95bf6e723aefa688b297cbdd", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/workspace/spreadsheet-financial-model-workflow-eval/source/records.csv": "430befe636c28a4b2751e842a79773440af0bd7a5f1b742ea177c6ef204d74c6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/103/review.json": "364448f195aa2af070f3a7ef503b5dbd14d8e213bda1f43d23d68c4b927878e0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/103/prompt.txt": "2e6e590bcb407f690bf427d857dc36b211cfa1ee8f86efee03872bcdd3ed92d7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/103/evidence.json": "83df007b310824f39c0ef4701208ab2cc57a3547207e23486555317e83a9de37", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/103/cli-output.json": "9e16d7b261312cfe591c22c094f969e1b0c2194d615d52fd0394ed62486d0058", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/workspace/skylark-launch-synthesis/synthesis.md": "73a2f8bfa43227b1ea5da0f4991edd34049ce94a558b97ab36660a7dc3856a15", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/104/review.json": "24c82329328af656e4d0b2f0bf9e5f55cf438d46afba4d1630f9d868692ea50a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/104/prompt.txt": "bc8fc89783598c2a21588eb38140c5c9c579203c5b2a6a9a1a7d515c96572960", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/104/evidence.json": "5ab1e1308f35fc1a3a5e5896d856e2b8e0946b9ad29dbde584b625f14ade83ef", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/104/cli-output.json": "0bc4f5a4dce288976b0fb4276903ece90b83efd1ef8894b8d75cacdbdea82b55", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/workspace/brand-theme-alignment/theme-report.md": "ea48f81235e78daefea01023ad040fb4526c609c9a686d23af4ea698975c3634", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/workspace/brand-theme-alignment/source/quarterly-review.md": "70a119718fdfd89aba5d0dd2bdc2e7df8fb45f749afc71d56bb5980e1fb030a8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/105/review.json": "bdb73d8a819511f84d09652a86cf076a643294bb1e6a0c68948693227032f063", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/105/prompt.txt": "55eeed51f12d06e480d13dcf45115e67dd5326a426614a002965f9cacd3c90a1", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/105/evidence.json": "648dbd7ee431127a4ee5f3823a9fde50f9c298a0e724a8a1dc3b7ba749baddcf", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/105/cli-output.json": "54229eef00d0d7bb5a3e2df2bbc25111ee2dc83009cb7837ebb077e27e1759f6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/workspace/powerpoint-grounding-eval/powerpoint/eval/deck.md": "6d91b1fad464c35befda51e9860c7d9b67e6f34a072d13d402e6a97c34fb24c5", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/sources.md": "6b76a2586cb691b681937e3e5fe096200abdb03cf4881f730ea1ad901c77ea7c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/outline.md": "36be00ab8c049cd162bff35b92c84544d8c3ec1e85d4c25f2d08b27a661979c3", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/artifacts/powerpoint-grounding-eval/tasks/1/review.md": "e8418311396b7bc958ba545280e739c01582d4bdd1194aea77111dde4234868f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/workspace/powerpoint-grounding-eval/source/brief.md": "40aa14b3b21bcd20520712ce43d510718f85a947768a87a5d402ac6b2e223c0c", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/106/review.json": "0ff44c57f065b296ce029fefc55cce5180295abffcedf71cf7bc6dd3fba43029", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/106/prompt.txt": "966b9e85aacf81c9cef2984db420a509c961b6a12a3d0c09e0416cc1667d8cae", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/106/evidence.json": "a355086fc1bc5933c2318d9ac1dbcc128466adfd9963e0db8840fe688613b0fd", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/106/cli-output.json": "687e22b0c84880535764a56152a32d7dcb8693ad083757c750a35dcf6dba21a9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/workspace/analytical-engine-research-brief/research-brief.md": "aea73a96ea8722143b510a4e621d006a880bbd7b846fcd913320c935677ddf3e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/workspace/analytical-engine-research-brief/mocks/services.md": "3c601a7396901d0030a995280a4b86afa0a89f5eaa5216ee956dc22e0a32e1cd", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/107/review.json": "cd0caaefc721f67b934490462c8e77cf739d414f4d83126da421c4de5d2bbcae", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/107/prompt.txt": "91e70dac90796f36bc3bfd2c6362182ff6877152903c9cbf8220549dcad8db02", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/107/evidence.json": "6d0fa8e824bdad30d7e20bf31442d93fe5fb266fb87c5c06ae26b56527cd7da7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/107/cli-output.json": "3156b6463caffe78a64aef17738dbc0699a938265d63d51aeb05c1c5ae8e6d76", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/workspace/calendar-desk-week-plan-eval/tasks/eval/report.md": "3a1e4370b4f333398f9a6efd20439ed5492d511e151c08b0e8026fb7edfdce7f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/report.md": "3a1e4370b4f333398f9a6efd20439ed5492d511e151c08b0e8026fb7edfdce7f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/scope.md": "ca5289766b251a105f761beb3232e8feaa0e613f7cabda7b037c37277f6df9f6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/plan.md": "deb2c9783658f0adb937fad2d52e91dd4de38c8383cd91f942aa6a5105db2c9d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/artifacts/calendar-desk-week-plan-eval/tasks/eval/verdict.md": "83e4cbfb0a4425e6e4210d4f45e7f7de72f462ba34febdbbca95831b981e4509", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/108/review.json": "f1f7b4923eb0302fee98e8fe99b16a0691f7c6da2d6c6964ecd248554925ec70", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/108/prompt.txt": "2a2acf0126dfbcd9cb2d719dc070a1d6fc6d36773218be544ac6023bff5c04e0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/108/evidence.json": "e0f8aabe02c5fa3d74d1ebead868d10adc9a6ff307f98cf981a01f39930168eb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/108/cli-output.json": "7f7bf381193a5cd7b0c2e347487e8b698378d3ce149837710d3fd4244cb6d711", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/workspace/a-b-test-readout-eval/data/stats.json": "fa1cae1885ba0eb4c7c636b4289e450ecee3742fd84775ddaad710d879ae51b9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/artifacts/a-b-test-readout-eval/tasks/eval/hypothesis-scope.md": "4bdced01e014a4510a6c0438ea3667a436e8cfd8f44ca06593d5ee0a126ca6f0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/workspace/a-b-test-readout-eval/data/checkout-test.csv": "4ea5f887445d2ef8a4e97c00497c7a7451f1efe9ebbbe24bcff5edf71c7a9b35", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/workspace/a-b-test-readout-eval/source/experiment-brief.md": "d202af80df2f81ffe067bf41b5999bb89fd107cd3aff92572b22ff612ce759ed", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/109/review.json": "529e15cc34e73e9f81371948fc9290fc167a1e5caa3ccb9bd4b644e8c15e9c02", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/109/prompt.txt": "39d8bb69786e5a0590b826a19b8734b3984db949f510e9b528dbc6eab9acc2f6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/109/evidence.json": "ae91b3059208130b3c25c7a599e928ef0f947eabf3bc4a2c0bb9b5fe10c48978", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/109/cli-output.json": "3cc583bc40696e4a2e66df3a4fe6619190df017e51b79ef05bbe3f70093ce7b2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/workspace/annotated-bibliography-eval/bibliography.md": "060796e9a1956586e31fd1cb8a941005c7ddb568c8ec5f09e86e794cf9183866", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/scope.md": "f341dad756907050a3322074228b8939a0909458fbd4c82e7ee06f4ed623449a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/sources.md": "eb5a6a77c40c335d48489f549a3c6ac5ac8ee553fe5f41dedcea98da57d134fb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/artifacts/annotated-bibliography-eval/tasks/1/verdict.md": "504d4817b4cdedbee23d40dc8afd2c6b2ec0a9e7d32f9e5123479793ef0d545c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/workspace/annotated-bibliography-eval/source/source-cards.md": "f32f3693d422382033b50e2e3a1c6d4fb10231fcc440f4dc5014ba96d3baa9f7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/110/review.json": "072456abc2771ff10bc2c8813485897dc3866d410d1b0785b4697a39cc28f5da", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/110/prompt.txt": "99f95050ddf84ea3765fe65f551cc7d9b33a5e5ad67f7b9b8d2f8ba0248c0c54", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/110/evidence.json": "33557bad3cfed3019e3e42d177bb3b7ee90f4689d4ff46c5b88bf67b4899cdbb", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/110/cli-output.json": "224aa2f973b0e259fe4cebcea4197a87ffcf716f247facb79b6f2bc53c197a0b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/workspace/word-document-from-content-eval/documents/eval/report.md": "c9625a9a378527befca2070b6e91770c99649fbcce3a7982d3529fea5792392b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/artifacts/word-document-from-content-eval/tasks/eval/scope.md": "a103153d9c52befe23dce5b02ccc62bdced1b229f3ea19d2440a343a920e3c64", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/artifacts/word-document-from-content-eval/tasks/eval/sources.md": "13efb28df29f1991b040effb818998876f341405f685e865c5dc84280c163c58", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/workspace/word-document-from-content-eval/source/brief.md": "54205b8339354be6f059373f5b7f5aca9cbdc6f3199fe94590642e11d8fe4de9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/111/review.json": "90a72a7647afb173e9a4622181ac491f040bae06a8251b2d40d9ad225a43e989", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/111/prompt.txt": "37170962453dda8642dd6214cb7f4bd47981850345119ea1e381e184aa6baf38", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/111/evidence.json": "bf8a2e5255b1e866137ad61e8dd83ef2469edb7424f8fce959ba88c55df48984", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/111/cli-output.json": "aa33153d905bf275134430641355e8fc0f7f05351856486437f729885823d5f9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/spatial/Geohash.ts": "29c40c6c7649ec7d2794d514fd31f9b422563d5a2076e1951737983ad89dc4e9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/imageEdit/persistence.ts": "93ae10d0b079c355a29b52b1d2af83999dfe4fecd38d6309b795c6987525c3df", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/workspace/squisq-core/packages/core/src/spatial/Haversine.ts": "6ba31700e271f0f3a33aff0339cb30e902bb890c78d74bb9c47f2e4b06eac635", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/112/review.json": "f84294570091fef29edb999aa966039aec7eba9005fca71f9d81dde9820d8a87", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/112/prompt.txt": "b936d60a8e8bcbdd52eff76ea2d73ff9f4f44c3f35d495fb51f8c0db3fda7738", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/112/evidence.json": "f64d165204999a6923cd1980c634fff43c3cf7669b1278827a4b224b974bbaff", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/112/cli-output.json": "1c23e279891e840a46b642ba97200e4e05d7aaa359d20ac9346afafce18aa6fd", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/recording/chat-events.jsonl": "943b9307f7f328b878fb5fca2f137c3b2ca7d41d73d842759ad14b8903dec35b", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/113/review.json": "8c52b99cf3c1e037f7a0582d52047a2f27c9f8540cbc6052a2164d01c2cea733", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/113/prompt.txt": "a7bfded13ec66a9238ac3952a0342135a802fab288584465280bceb8684d6779", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/113/evidence.json": "62de620f386ed5d3e4be15f01981dbaec2dd3b2da53ab120be5b21a2d71e762f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/113/cli-output.json": "bb8b0bd791d961e860e215ac6ff6c6bb8e0af4565d4ed3c0390e520e27ac077c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/types.ts": "69418e415dfb47d45a850ca60d9a455e7bd1ba9bef822e4b0ec004989b5bf474", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/producer.ts": "a3b092a44698f23dd61016f2237b017c1911a0c80c29e7a04671c7cba2a32de7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/workspace/event-pipeline/src/consumer.ts": "f66d051a37e824e8ce1f79da4e56d184826809999bc9564e3d028e8323175dc9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/114/review.json": "512db666a5b33b70cf0748e335f34561ddbb00f7c3eba035644d0c57750f77c8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/114/prompt.txt": "4494cee3a7c98a2be409d450fd076cc698688ce4988abe2f82b46c756222b2e5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/114/evidence.json": "cf2cf7705d56ac6626b46eea8954e595b2102a479c34f758b68b061e392552e4", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/114/cli-output.json": "12f2b600d61178526d604c616127fa2e235062dd206ffafd96c35378b91733a7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/artifacts/code-review-workflow-eval/reviews/rev-eval-1/manifest.json": "81b2d90e442e6b7c24fbeac782ba7abd0cb5e8d2db3921bfa179638cafadd47e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/workspace/code-review-workflow-eval/source/craftbook-eval-brief.md": "5fa93f08622aa11e437b5b34aed4c7e9af81356686544a190ab7b2d77762cc16", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/115/review.json": "26f74b98d09a443e3f1c2d66a68bebef6600f5206859db8f284c6469955254d6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/115/prompt.txt": "01e81f2284b48f0536d10fb79e45b260283c5763d7bcd095ccd78ec300868aaa", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/115/evidence.json": "dd4f7d95c2e3467a394f487be60a2ba853424c7b44c4b76dbc88366ec4f3c072", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/115/cli-output.json": "321cdb55b4184529e58ac6b8f59ddf5d4508d7349f539279ee9b489207facfe8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/artifacts/api-contract-review-workflow-eval/tasks/eval/scope.md": "6e1de595b2c87e7c4cb64701806f7671262abeef2edb3c351ee72571bc7253cb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/workspace/api-contract-review-workflow-eval/source/craftbook-eval-brief.md": "8932755301918adbe04baa72b3761d252c4019d5e5c84d46999ac3781bea7a97", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/116/review.json": "caafcdfc97ea516c38731210aa7752712ae486a3b942170540f7670285077789", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/116/prompt.txt": "bf84c347a201e7863d42367060f79b5b97c58165f290e875f260eab1b347d2e6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/116/evidence.json": "2b92ad0fdc6e3bbf3733412493b4f157f11b22d3b94e406b781ee6d161cd69c3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/116/cli-output.json": "a156b27b4c68ff5a9d311e2192ccd42800b1a8166807ed9d6b0d6416b10695cd", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/workspace/security-maintenance-eval/security-review/REPORT.md": "a0fd1fb60fe8925aa18fa5a363720d665180664621a955fc6a9fca71ca2c814b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/workspace/security-maintenance-eval/security-review/findings.json": "30f6f62327fb600940d6546ab05d94d355f7b71b5135db88c13774360a68366e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/117/review.json": "dc7c4f0a09d4d24319031705fe90a0962acbbf1ed3ad58440f91f5c10933b9d7", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/117/prompt.txt": "420b3baecca73f8980a91f876bdc272cf16091e08eb1317358afa3531dbb0f60", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/117/evidence.json": "560fdd306e37c0a27c3080cd15fd6dee9b778c9baf813e43041b5cae09d40a55", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/117/cli-output.json": "b66d8119b4da87e220ab60e1256c5f836e5a924b745108f2783525a27a51e05a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/118/review.json": "bd9ebb4a68c37ee627798d8b62f4e821cbc30dacf00fe861dd4d7c04cd53dfdf", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/118/prompt.txt": "99b9d53633893a564f78da7a903df1356239880c9f5617c5691245085cd4940f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/118/evidence.json": "ff2f76e287c4680708c5bafb71cec29c8b887a809e3793187e84b981497727dc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/118/cli-output.json": "7d09a9433fa04005eff01433dffb2ac16df3158386e0b5ecfbb077ab05cca252", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/verification.md": "e2fd8b7557cb66911e7f6ccf31866fe5fc4cbf1dbd55dd6b55190a43764e5771", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/sites.md": "b8f1af8b1d652462da4c310953ddb5545156f7d816fc2240ef79c15afba8b6d3", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/artifacts/profile-cards/tasks/1/change-notes.md": "7775292e85e003cc5f9aaae107200ceb2eb5cd18d459cc789ff8c23679222dab", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/api.js": "bd78e33b9cf74aec788b859016f4a43e161b0173bd06add93ecc140b791c7a0d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/index.js": "5619055c727dd95badb52f2a973c8d0679a05b006bd30d38c3efb809e2ef389c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/src/profile.js": "fb079d5420e1e811a50449f8b905369da1359f6bd1d800053b350e323b763284", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/tests/profile.test.mjs": "1f687e56bf6fd264f98007c0fef446bf87f248ab076394785ae5145f3501f9a7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/workspace/profile-cards/source/rename-brief.md": "3fa8c5e72f003f711de1c4462a4533a44ca836caab9dfd8b11a75abad6c32d80", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/119/review.json": "15258f433fa66b8e4b6d3b1cd3e04fc09ddefa6bffb282b4a40dd05b492b9bf8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/119/prompt.txt": "42f5ed45b166320ff0ac45e0c0da24b9505a1689c5b35d84d3b1a2f3e9c057e5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/119/evidence.json": "9818ba8a9f38500c03ff44d3cc081de45752da5f9abbc121fbae2af5bbad6b6a", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/119/cli-output.json": "ee9ad29bc6fd143aa5208102865ebc967f0a01d2e22c5dbcd75525955d1c5554", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/artifacts/crewlog/tasks/1/measure.mjs": "7f2ddb4fae250e215a0ae9878837b653a19eac3cbcee06127750feb72fc5e470", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/.measure-baseline.mjs": "7f2ddb4fae250e215a0ae9878837b653a19eac3cbcee06127750feb72fc5e470", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/tests/characterization.test.mjs": "04a801e3542a014a4f26b3b4c34904dbcf8760938c2b54bf4b9a0cee82087d19", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/tests/measure.test.mjs": "212d1175e046b301f92e13295c83ef68e27263b3a9134833b019abaea58fe16e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/workspace/crewlog/source/refactor-brief.md": "2aad38fad05386bdbfc74c5d02a2c1d787ffa10fed2b97f25837e124f36d5d34", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/120/review.json": "08fcb57cda8f80a4c70c65374b7c493721a332876f71037d6584d0c9fd7b52a6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/120/prompt.txt": "521b5b071e724478ec1c72e619b74d76779ac1c36b87721dfbd7c5b7b6afca37", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/120/evidence.json": "a400ef48153bd0366bca6176f57622cb537196e85beb5c7ad39625e16f51d5c9", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/120/cli-output.json": "8cebdb4e34077ea61d95746ab60cff1c40c6013d4e6fe8bfcffde17952064449", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/fix-notes.md": "e020342999a81f677a63fa533e74a4ca73e71ec0ef595345ba2d4125568a8404", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/repro.md": "0d89f4890d627f4ab2ff9d270ab7a94b44a2d54315d11d80abe09b2ca27aa83f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/diagnosis.md": "19a1db56dd00827aa5415cbf9c4aad404af175e03881858ce25dd553bc490ea9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/validation.md": "8241254f3335b568bd5ed13719e1f3dd0750d70268e33b82a9fa7ce4ce4e8ae6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/artifacts/orderly-pricing/tasks/1/review.md": "9b6e6d60a4bc6813acd16369688c17a8fc022f80f837e3842b9aa30a099bf3b8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/workspace/orderly-pricing/src/pricing.js": "fad23b0c7929def833f4dcb9c3557a77ccb3f3ad92b3be97e462fea4e0a56921", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/workspace/orderly-pricing/tests/pricing.test.mjs": "59d0ebcb35b11217b4b68e85aeedec267e2471718314f2eb8df2325724ae0d01", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/workspace/orderly-pricing/source/bug-report.md": "8099cba5beb9322f507a64d58e0c3c322e2ea48ce074181877969ac6a588a43f", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/121/review.json": "adada1379d22f818f0bf1da6baa06d8db4bb2f8ec46d297a2e021ad0f6dbbe21", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/121/prompt.txt": "8fe72ee5307d558b60510f02edefdd222505cf967ee1742f53588100f34c9e77", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/121/evidence.json": "1de0c7f11472a2487573303a94f45c446b726f767607c1bb3ac2301b7525a862", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/121/cli-output.json": "0f81184c2a3c0e3eaa96b0f0a98b0ca69a988135d8837ec858f932d22e8e1e19", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/122/review.json": "669d35b8cb75d23d6a1cbe64c5d64cf6fa2bdbcbd4be782fd08b0a30fb774cde", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/122/prompt.txt": "78f378dc6fdc73a06ddeae011947e8eac3b3e7c7f6d6eef872e6fd0b018cdc33", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/122/evidence.json": "13d8cbd3ac4c06f6567d5d4f3cd6205da1b949c1b84b98297a55566c2f331149", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/122/cli-output.json": "4da63d8c785f99afa402db18676c518aaaae4a5df25eb27dc06e76c00411f0cb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/workspace/quarterly-brief/data/notes.txt": "949c911a4bbdf1083c68bbe94d539edda30128a353f4028a07a65bb717752a47", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/123/review.json": "66103bcf9ac5f0cae12d76959f269e1e0269cbff9d17a87352b86273a11056a0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/123/prompt.txt": "8f397356a7ec750cc169bf4607f024caad9ebc5d9cf1493b3133193d4842895d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/123/evidence.json": "6c2e631e86feaf898e81d0599b7a0de4789481fbabc20e7849be904ef989e7fe", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/123/cli-output.json": "3bfd181cb6b1080488a83fd0c599c80eea28f0c206d7cd69a425179058cff226", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/report.md": "279e90551f2de96f799902c411c0f98fb4611664cf557a8441082fd3d19001af", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-042.html": "c9e5993975521e99711b99ada34cb84b103b7f75cc769e5d1f993aaed52e6712", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-043.html": "affe16ddab8f14c8ffe13f8c5cd8569e99d0f5af95acb19dada69dbe3c2f1d44", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/invoices/2026-044.html": "7eba50a1c71c5403ff0e279780f0cacde72b3a00cb3bd5ee6b4dcf9567b1590a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/workspace/fieldnote-studio-office/ledger.json": "f73aa32f527317a392d4513747a8176870341de144ed77942d1dd92a6a9c1073", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/scope.md": "b818836fb6ee8dda8d899b7aeb0fa7287410b9a96e9f536690161510ea1188d8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/billables.json": "4513368faac4a8b19e00e6ce11e00d9036527597761966314db708265bd735ee", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/artifacts/fieldnote-studio-office/tasks/1/verdict.md": "1727e7dda22f9942d7bc6d1b23e0e87483aec0a37726fc1233333505103b11f8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/124/review.json": "56191a364b3d1dd71860585d3e8e8ded8c583609331491888de10b5cedfc2c8e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/124/prompt.txt": "5ee224cc81bf330e050cc3ddf64cdeaa6ff3f3b6d5e91682189362031a3b6ae6", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/124/evidence.json": "bcd95332ff3c48ec96b0d8da00d2b862bc6becd7c6933c7d003515436c896999", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/124/cli-output.json": "666af88435a96d34674fe5b62aacda4087f5cf0f0eb05f477b0d3792953d6f8c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/out/report.md": "46477033a36d64c6fc5be3caf52f40b60241673eb788ff0dd66afd5e349f7851", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/notes/anomalies.md": "70e012fd3f6d32ccc48126aabb5932442a1a8fe1ddeab437d66f4d97ec74f601", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/out/orders.json": "1b249c9142fa797f5ee04c47e946499196fa284d603d1a80b87707558f233410", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/workspace/order-intake-cleanup/data/orders.csv": "c88785ed78c4d8b14484d8d0f04516c9c93c99045b2e90b60b19f96ee17d7bb2", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/125/review.json": "dd5e02cf1b9d229e45645743f4dc22c36f709ed46f9ac9a27682528c0c91d225", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/125/prompt.txt": "c307dae1f2e471121712f7f5c4fb7f0b308d2c34d6a9fa597dd83ece93afce39", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/125/evidence.json": "d82d1d4fd40c8121513ee250aa1253eecba7b5aa657239feedc78f99794dc2f0", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/125/cli-output.json": "47b4d181f57cf300290ba8dbf7f80700c337e75494c029e28441e75de0d668d2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/out/press-release.md": "f532e8eff0c9e6c7e5b47c68d3e4870c6bd7b7a12ce3493be7ec242e724f4569", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/press-release.md": "f697ad2803394b9c27ab4973c20b503ba629389da497651a08c7b002958873b2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/workspace/fieldstone-launch-comms/facts/announcement.md": "6e19552f74df994beccd4b45ff25a0a8feb92aef7ce5e918782f38ef22d6a39f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/artifacts/fieldstone-launch-comms/tasks/1/scope.md": "59984e1d0d3a23fc89ae2f2dcfb668be396cff9f288c4750a0293ee728029f15", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/artifacts/fieldstone-launch-comms/tasks/1/verdict.md": "4d87cdaacbc894d2f57108e91ca73ed23309471f8435a613eb0231f9b4a41e5e", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/126/review.json": "c32dacff220ab8fc77f6d289b5c6a2c282c3a265cd2499583d45fe11a29eb99d", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/126/prompt.txt": "27dcfbfa61eae7586d0827c0dd6f16d44a0ec6876f830e8fff84dce1d9536774", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/126/evidence.json": "82afa3708f56895071bc6b0476313c80ba11d85b08154fcacc48644685a270c3", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/126/cli-output.json": "1e7282cdd300cf20dfd123e0006c9cbff27682d88f62e0c237157fd167626c42", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q2-intake.md": "95287311053075cc2d18b87aed6e8682b26ede21e1310d0cff30608e8bd39467", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q2-clean.json": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q1-intake.md": "31380a2646fb01b6026d51f4ce10e5e9a37fc8b09a11a163714654b454acf9cb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/out/q1-clean.json": "5afd1ac820f6351024084c81a0821378c962dbc520a5a0c90b987645f229138a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/data/suppliers-q1.json": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/data/suppliers-q2.json": "afaaecb3affc61d3bf882dc328e6072086043597ae3f1fd113c60be200bd0c91", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/craftbooks/supplier-intake.json": "b5537301eda4b706a5c718155fb96334794fb40d407ed9e6e112d835a44ccc45", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/intake.js": "3278416fa9bf743d440f925749a5b3d12ce90ddcb97e09f86ba4f5b5c349a64e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/intake.cjs": "9323ab99791cbb17f70331e97ee8fe4cf73ef078dd96a0633771183878b3abf4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/workspace/supplier-intake/verify-q2-clean.js": "1fa3259da0fb0887ef7cdf81870b0d6cabb4c4dedb476920585907b33f655980", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/findings.md": "f69ab37771c35fe8de965e7517a1892b738edc36513b30bc5abeb1d594b1a389", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/out/q1-intake.md": "7002c8fc9fb57c7a18ae7a37dea0887c20a19780a7369364bc2e7d62f36a7223", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/out/q1-clean.json": "213f13cae0f153dc957fd7fc59e34e1c74fe6af15663f646636f319f8d0a43a0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/artifacts/supplier-intake/supplier-intake/q2-findings.md": "436f5184875b029cb6d53d94811a37f048acf1f8ae4fc8b918edd9884af95958", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/127/review.json": "1777eb84aad930812ee8a41c7e74835bb5ccfc3c323e36d04a8c2a2e7b45b183", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/127/prompt.txt": "87a8ae817d437fa761fe323deb1e71d07586bba8ad21e090ee77c35af10f2bdd", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/127/evidence.json": "a36312bbe5758d7265ebad76076625de22961ffcdb677427b8244684a0195fc8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/127/cli-output.json": "c7503b6625114ede54a76d3725d9deeb7fedb56b350901544b3bf375477774c7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/out/north-summary.md": "c9ccfb95d076c62d7595b4ceb8151bb6a4e95b83c1212f81abb721b87d6e4ea8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/out/south-summary.md": "4679d8bf920313c80504983b8189dfc9a0fe9ade73c573d0676b399e56d3310c", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/data/north.csv": "1d565713b89e9dc055719ef769bf6ba5bea1ff888d309aa34767a5231a0af9ba", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/workspace/regional-sales-rollup/data/south.csv": "e68e801e3fc4700a2f4e7625ae8db017447668c1b36b0d19858bc309ac0be370", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/1/numbers.md": "930c603b8ab05c4bae46dbf8a80e7899a7851e659fe372d7f79d80fb72582b72", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/3/numbers.md": "320b444a78d89bd94a1eb5ea2156d2e645a67957c1bfa63e78963d5537ce2e14", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/artifacts/regional-sales-rollup/tasks/4/numbers.md": "0315e9c3fd170b8258beed9a57a3a491e28f7ef022ad3a0c0db31245a455e6d5", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/128/review.json": "636301161a7d0066c150d5f4dcabe9ea060d56e8317d80ce480c747cf4dd7e60", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/128/prompt.txt": "7d1632881b5df62f93ec0a8c0e21ac245cac5a450041bed7045fb5ecefaad8cc", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/128/evidence.json": "258bbc330aaf91e671aa3db57fa6eeb9b08e5c1e2db1a4725b17329db62b8bc8", "evals/runs/saluki-native-0.1.57-2026-11-03/claude-sonnet-5-5-review/128/cli-output.json": "fbc2c7a9344c25fe7481296f745d11422dd435e240dcc1bae76d0aa2479e75df", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/workspace/out/inventory-report.json": "875d410067ebd445fa86631d9ba7c0b171c32a9f2411174ca028bf0af43950be", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/metrics.json": "3c02d77c4ab2babc39ade3657f4f92a8d7507d2c4653cf0135b26b9a0b2e53b6", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-04T00-05-25-964Z-gpek/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/metrics.json": "195c3f5cbdd7b85c58ecc9bc9a80a5820e228762bbce0044527dbb1b696532ea", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-04T00-16-26-050Z-1i5w/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/metrics.json": "fe5dec657222fd8c167a8df2f8e907fd130c816dce0e8691d97d77e5493bb8dd", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-04T01-40-41-736Z-5r29/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/metrics.json": "93c97b08b1b0e4b0e243205441da792d5f5443264aa1a4a220cc46ea6d612058", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-04T02-13-44-132Z-3p7z/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/metrics.json": "9d82d5d02552076c95854e7ee5b6fc706cd04ef2b58b906493602ac333521924", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-04T02-44-03-484Z-60uo/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/metrics.json": "b2ba10912165240c080029346f4e48d93e5d0040f5cc6ca161518bbd5b7168a4", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-04T03-05-54-026Z-2pi1/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/metrics.json": "8e8abb14858ec53d14f6525cb1a73ff491be08c72c4445e0091462c77b98e373", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-04T03-15-59-497Z-0stw/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/metrics.json": "92580818fc182887c3ebc584f394623bde053dec7a95264e4d127294e3fa6a22", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-04T04-15-17-173Z-0cek/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/metrics.json": "3096f68d88d45e73800ed6bffe9230792e2c67233cd316ebd99a311761515dd0", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-04T04-51-33-586Z-3pp9/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/metrics.json": "836b7d07d067af86db1af7a1b5e3de512b2d905213b5bea1eb076c94e383683f", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T05-06-14-544Z-4tg9/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/metrics.json": "7a0c8c4e6ee1e22bedef169c5d2e0f1ddc6d39ccaa76dc38ed9c640255671175", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T05-17-18-555Z-4jcd/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/metrics.json": "3637d20ca5c7866e5aae50962e5ab57ec19cd65d737c1a4a156ca09dfc3e455c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-04T06-15-01-377Z-945b/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/metrics.json": "dae179733107fb2ae4ea2fcb61f848c4b874e3faffcca1ccb17f6cc06a4ead06", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-04T06-25-47-184Z-1jry/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/metrics.json": "03288698907872ef16f1704b987656f2231b8389ce5cfbc3a3c522ad49a6afae", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-04T07-20-21-294Z-4kek/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/metrics.json": "2266c2390f8f501b33488e80ce655b52e99eae30c4f010d883ee2176b42515e6", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-04T07-31-51-777Z-429c/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/metrics.json": "cf64d5ab61dcad7dec99073b023e1ee8b3c062d2ff289d72447c1433630b42e8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-04T07-47-54-584Z-8tqz/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/metrics.json": "0ebd6e19ba916c6377ba1c9b83dbba094195a1a8d4e43334ccd5855fba4493d5", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-04T09-08-33-934Z-i0ew/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/metrics.json": "f7fe8ac850c17efee92d7b85dd606026bc79a26445b841ab01161e99b90eb9eb", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-04T09-39-57-681Z-tkrr/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/metrics.json": "82ecb9c789af087b07ee40b6c11f4c2024837c17f4ada0d02beca0514c52d875", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-04T10-01-30-180Z-gc31/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/metrics.json": "7c956ace0ca12e0ead35b73e3266892b06c7b62e8dcec8211949d9f355d99b75", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-04T11-17-39-776Z-5ncd/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/metrics.json": "1efa5e105c8d194d504df7f20c1e4b6873f26f89ef38226938a31ac212d13345", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-04T12-04-36-260Z-bkxi/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/metrics.json": "e1d4c7ca220b49ac5c79973626f470958fd87b45765024fcf74fba993e8fc2b2", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-04T13-40-24-956Z-dbhi/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/metrics.json": "679be741c12da773a8034de792dcce0ceee42c2a799e528d09ffd7be6d043d9a", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-04T15-04-12-085Z-9axo/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/metrics.json": "2d8a79b8a83efbd65e3907497c7994e876c37853b83d600c8c2b708cccd1151e", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-04T15-28-35-768Z-krj5/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/metrics.json": "06c2e9c1cfde20aee08fa0a62d688912aae949ae22a13cdcb7dd31c105ba0811", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-04T15-51-11-509Z-0c2q/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/metrics.json": "c8062880ffd689a5467e842841b33add09e6b49377a00c175f1c70acb8cfab31", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-04T16-08-27-562Z-yxbe/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/metrics.json": "448e0344d3b55196d29356255d3aefc174c40ce53e253026ef8924273df9cc8c", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-04T16-12-00-765Z-8y6i/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/metrics.json": "8666f774ca49c87e505a30317c4534b9126dacf3c8d71b05187b6223adee21e9", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-17-39-748Z-7wv0/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/metrics.json": "83ecda39333c11911183bd7a16195cc66ef6cd6a5852bc3823de0c0387fcfb03", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-04T16-46-58-460Z-lbm2/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/metrics.json": "94a56fa1e13e5d0435eed3a6affc38976dd8b8d89c5baab8d4c82a430beec524", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-04T17-45-22-802Z-hot9/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/metrics.json": "2f002bbf17ba72b6ddbca71443c36bfb1fb13659bf61035b49f1090cfb35ba80", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-04T18-07-39-731Z-bpdj/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/metrics.json": "adbcc7f69c92ff599f14ed1a810a56edce171b0418d950cfc5a10793f5b9f089", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-04T19-37-25-765Z-tdx6/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/metrics.json": "5298328c030f14cd8111de8c37fdcb684365284564aae521b341bc677e49b8bc", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-04T20-08-16-625Z-9gn5/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/metrics.json": "b853c68d9e2ddebcb0deba8a1cd484c99956e7fa1c88d2cca28657502760ce09", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-04T21-11-12-350Z-8q4h/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/metrics.json": "807695963d83dc7ef6d1a4fc580fe76c411ac12f34cc16a06ee54f821103b258", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-04T21-37-06-556Z-ntj1/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/metrics.json": "a68cb691c8001d30c5346d5d9f3401a44aa18ad55bb390e936f539107b49a2d8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-04T21-41-20-383Z-kl8c/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/metrics.json": "7f8c42288cad4f79743e21d9e9a1e1780198d2ecd9b1452e5b89a01717a9a4c7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-04T23-03-24-956Z-bs2g/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/metrics.json": "9a15ef91c0567862ed996195aa5258fd863c0bb9dbe594b7dae4898265ea8957", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-05T00-05-17-288Z-7rlz/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/metrics.json": "a1e9aa7c7c9dc199fd989053b14ba175b29d3e1daf3b03d938c24d7886bebfa1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-05T00-50-32-680Z-exqp/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/metrics.json": "9a452ab92664dcea93806922eb090c419dc218ca42f059b2ffd518dee3d563a7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-05T01-25-52-407Z-2xus/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/metrics.json": "d261ff22d607bc58894c6c23e09f3c70a4fde93d059e1ff91f7ef26618585856", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-05T02-38-35-027Z-ok2p/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/metrics.json": "0c08b0d26e5f2943311fd7b33cc667f5e1bb618874d76eec356808ea66103eb8", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-05T03-15-52-588Z-x1tt/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/metrics.json": "6b91019623467eea884310dd00931a2ab9bc4f3dcb5dbb3bc46dc50ca6920ad1", "evals/runs/saluki-native-0.1.57-2026-11-03/r1-saluki-cuda-complex-work-craftbook-author-fanout/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-05T06-01-02-970Z-sx04/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/metrics.json": "85d80f0d3e8867b8934a1023e8aa4bf57815ee1e14973351b0065a545f43616c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-05T09-50-34-101Z-t9wc/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/metrics.json": "d979841ef3cec252223eb4d620dab96b9902e2cb2871fe13ad3b775720ebeb5d", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-05T09-59-21-724Z-fzhl/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/metrics.json": "910ceb0a7db42da97b9a41ef7d459034520996aa2342a00a965d1206a353c009", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-05T11-05-34-374Z-2vk7/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/metrics.json": "9c03fb126914bf6492870ad81cd90e30d378c18a3b80d8ea6998499ed313e3f8", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-05T11-15-55-017Z-8hou/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/metrics.json": "0299cd38565d697a544481272cd0b73f0b746aeede314ac15514418123caeaae", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-05T11-27-25-395Z-wslk/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/metrics.json": "10e445635e4b9ba055830e6b3db271fa95d1151a17bacd0dfadc677824e9f08f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-05T11-31-24-504Z-k0px/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/metrics.json": "7dc285930fec4a4be451b2cb72aaa3a9abddcc37cce1b9a59e8cb21526909c4c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-05T11-37-37-260Z-1m0q/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/metrics.json": "b82231c2cdd8c62301d52bd953d78bd229782632931d81c86e093e16b33174be", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-05T13-26-14-307Z-y3do/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/metrics.json": "67ebed1f7b35f8c1275833921988078b73196b8470f388bc153c5639fabc753b", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-05T13-43-51-252Z-xif3/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/metrics.json": "10fc8817229bd9d9e4d03da46c6d8b691b5365ba37ab1f2d50d1dd6427a2e6cc", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T13-52-18-463Z-3yet/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/metrics.json": "5d5dcadef639bd6f5ecdc91fcb86d6ce7773d43eb70a55aacc630566234e7ba5", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T13-58-08-649Z-glwf/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/metrics.json": "980fb488ceaf7e8f175a1c7352df96cc928fe8e0b4d2857099a6a35d8b417729", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-05T14-19-48-929Z-ych6/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/metrics.json": "e56fad8d65274c12a37575fb9a97ec17209d44acdfac9d9571fff8c43af17e93", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-05T14-25-48-976Z-upa4/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/metrics.json": "6b7a6f98161071aa0ccc58fe408f105b391e7e98713e67728bd36e21e0609577", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-05T16-17-40-854Z-t5k0/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/metrics.json": "3a57aa9ca826bbf9003c2000ec28852d91719ddf01f60781c42eb75773330c34", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-05T16-23-32-328Z-45gu/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/metrics.json": "faaa14f777675362e7d5c0758a9d3fac9a1a8d049a80d90b4aada31a4a618a76", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-05T16-31-03-218Z-d6a1/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/metrics.json": "fd20e699fc067e36f6a0abdc0492cc1f9eb19c6ba9e0d2611c1a7e876879ef64", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-05T17-48-58-017Z-tjqw/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/metrics.json": "2c960817740af077839df11c8d43fb8d6868aa2f54193112821083b43c1ed3a2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-05T17-58-27-608Z-0dux/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/metrics.json": "a5fed5b0a771750861940ceaa3694a28484317b6ec95c2904bad9eae807d7138", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-05T18-07-11-969Z-2z59/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/metrics.json": "e86a9c3cd8519557de143fe590cd670d0949e5f65e655955ab9b1ef69fcc7b82", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-05T18-42-31-349Z-vafc/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/metrics.json": "b3aeab6f6c85a5f9fb9b21c695d902bd1ff75ad5231751adfa8f29cbaf7c3fbf", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-05T19-02-34-585Z-ivl1/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/metrics.json": "cdba68870a1327ae1f31a6c9abf9c5b47070525cbca7caed97c7e6b6ec7df8b2", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-05T19-37-35-761Z-mx18/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/metrics.json": "9426fbf59b6f0603eaacfb920dd00b6605a719e49903d306b5c2d6e678ffd0a1", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-05T21-35-05-918Z-odm0/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/metrics.json": "f6d5fd274e2fc474e8ad41ca6030a199cc4135db71d729c307d6e83066576688", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-05T21-53-08-912Z-e3h0/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/metrics.json": "1ce6d34c37142cc9cf9d2b9b0d8f1ac342159935120800dad504c1e5968b1aab", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-05T22-21-46-180Z-mkf3/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/metrics.json": "0af0c16df3a3af632658193a67135098b0cb819415d0cb79cdcc324765177b92", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-05T22-32-46-672Z-wvwm/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/metrics.json": "acd4556a4ec7b36ea409c7d86296c63d9914c4a5ee90928e44a1f86a4a4ddb44", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-05T22-36-51-364Z-ii5y/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/metrics.json": "f4d428e5a3683c50d733442d882c4ea29aa45db5bc25f8fc1f5648ff85a00114", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-05T22-45-20-930Z-ulff/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/metrics.json": "a723efdf7ed661b54ec2bfd83ffc5696e7919084b2c469ff51136a9b0ac2b6ef", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-05T23-05-09-444Z-hk2b/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/metrics.json": "42158967519ca301ed461079c7831660434cb2a5d302634a20b0495267b19a68", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-11-01-045Z-x63v/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/metrics.json": "5103215571e9df93a515c1989a6228f49c02655d077a69c4f7e3dd7cb1baddd9", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-06T00-47-11-236Z-gq1s/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/metrics.json": "d72428e152df2925232771d8924b4a485281fd449c163c68dc1cb4454ddde7c7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-06T02-32-28-346Z-cu6z/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/metrics.json": "179cc7705f8d7c29c7df81b2fe45be82e682d6c60d2ebbdd809468f70f83889c", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-06T03-07-31-670Z-u16c/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/metrics.json": "82a39ef06e847537ed2661bdde7c0b7d430d193dd19c252ab573f53b5f074059", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-06T04-12-52-338Z-lfv4/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/metrics.json": "81732daef9918acc779166b7d46f7d42cb2d316f921d38944c103d0d057a5f48", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-06T04-40-51-154Z-xkhe/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/metrics.json": "70967ca45eeedca5356300be3469f1b96462dda15f73bd1decf3d5b0ca41c422", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-06T04-45-07-147Z-ymas/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/metrics.json": "319705c132723a2cf08137b33fa549863fd998a855f2e428b2c748e75a8457ec", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-06T05-51-45-280Z-gkvw/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/metrics.json": "7c404e9bbe34621e0088a4c660f3a059941d90862b80c9459e8de5651535432e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-06T06-22-58-627Z-pazz/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/metrics.json": "a158518b0c798174f833fadcc126a79855f5eea1f4a454863ec4c9289f1a4971", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-06T07-33-08-377Z-788t/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/metrics.json": "d8c2c1a00e7f78fc55e57f2d9aceac55ba4abf2ad5468e21c361cd079e4c016f", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-06T08-10-28-851Z-anv3/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/metrics.json": "2b755fc93f8292330f892873456d3a5f9325aa57d25fc68ec1dcd6ad4b7ea84e", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-06T14-54-26-198Z-9jmc/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/metrics.json": "7805a2a3b584164f0c4cd54e9d5cde2a43773ec4c86802001df4d16bc08ec892", "evals/runs/saluki-native-0.1.57-2026-11-03/r2-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-06T15-16-17-977Z-jsdg/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/metrics.json": "b38aa8af21b2d372e8854cb24675a749691dda5bc5b5ca93abb3ba21a2cabfe3", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tictactoe/tictactoe/tictactoe-underdog-saluki-27b-iq2-mix-2026-11-09T20-16-26-587Z-xifl/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/metrics.json": "56a13bb70bdec86c0b4fa780799ef1c0ed9305af931db0908bf54c6093297fe4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-petshop/petshop/petshop-underdog-saluki-27b-iq2-mix-2026-11-09T20-25-10-813Z-b8dy/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/metrics.json": "e0030058e552325fad60e9a104b781f8de2997a5bfc338ec2c64dec3db91e9fb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-tankcombat/tankcombat/tankcombat-underdog-saluki-27b-iq2-mix-2026-11-09T21-14-42-224Z-41w8/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/metrics.json": "3f1b88547f6fe1a05307fb7dac94cddf3d65854f0c4c74b7b3ea087e30fe7b5f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-schema-migration/schema-migration/schema-migration-underdog-saluki-27b-iq2-mix-2026-11-09T21-25-46-082Z-a203/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/metrics.json": "0d61e560fe1dffd84a3963f224476dd7abda1d4d3027f2de18faa8ba02d8647d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-failing-tests-spec/failing-tests-spec/failing-tests-spec-underdog-saluki-27b-iq2-mix-2026-11-09T21-37-05-190Z-4vve/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/metrics.json": "99bcb4054e6fb12ea805038a93ac88fac15dbd03f9bf30dee11050c70bdefee6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-symptom-debug/symptom-debug/symptom-debug-underdog-saluki-27b-iq2-mix-2026-11-09T21-41-18-850Z-2jfn/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/metrics.json": "e3b1b232910c32442c358d44264d82a61d3b114a96a7e2041ab63018378b47b0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-data-wrangle/data-wrangle/data-wrangle-underdog-saluki-27b-iq2-mix-2026-11-09T21-51-35-159Z-obd0/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/metrics.json": "4f3537249358286ac98b8b3d1b061937e03e95af8afcc91c2ee7bf57f306ca4e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-incident-postmortem/incident-postmortem/incident-postmortem-underdog-saluki-27b-iq2-mix-2026-11-09T21-58-34-962Z-p5hj/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/metrics.json": "712be8faa14eeb08b596d59bb9e06178f9b786748d2ff0bcc28dffb3e0bb32ff", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-ops-runbook-anomaly/ops-runbook-anomaly/ops-runbook-anomaly-underdog-saluki-27b-iq2-mix-2026-11-09T22-16-29-575Z-tvu2/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/metrics.json": "9b5a4945562f517d4fc08633ed5e139c93f195d5d65492642d6d9fa501990da2", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-09T22-23-01-777Z-g3kt/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/metrics.json": "158bb36fdc2d3c8bacf88bcecb9786cada7b9cd21767101aa238da8cdfd0b7da", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-core-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-09T22-28-21-713Z-3kl0/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/metrics.json": "c01988be48f1e75247bae010f68329001e050f4dca31959a39e0b8d968fb8ba9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-constrained-comms/constrained-comms/constrained-comms-underdog-saluki-27b-iq2-mix-2026-11-09T22-59-25-389Z-yth7/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/metrics.json": "2a02e4e105781c932639f0c64120cba94addeb7a4ae904cbac998e363e21c3cf", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-records-intake/records-intake/records-intake-underdog-saluki-27b-iq2-mix-2026-11-09T23-04-41-731Z-tlqe/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/metrics.json": "2ad812499dc8751caa0e15c2231b51426bef29931189fa6e02b5fdeabcc4b77e", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-plan-and-estimate/plan-and-estimate/plan-and-estimate-underdog-saluki-27b-iq2-mix-2026-11-10T00-23-29-233Z-ul8o/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/metrics.json": "b8777ec80531687b299f0b0ce2b906c36357069d377df5148f63c98d445092c0", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-meeting-followup/meeting-followup/meeting-followup-underdog-saluki-27b-iq2-mix-2026-11-10T00-29-08-898Z-1nl3/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/metrics.json": "1d167ab5c2ae542fa3c796ccb2c79690af4a52d8549d5cea57b41d375c1b3d07", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-spreadsheet-model/craftbook-spreadsheet-model/craftbook-spreadsheet-model-underdog-saluki-27b-iq2-mix-2026-11-10T00-36-23-881Z-33tc/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/metrics.json": "8b8e7f282d46b2997f4249c47d5d9b1d54008cb7130696b8695e5ffc5c0d98e4", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-conflict-synthesis/conflict-synthesis/conflict-synthesis-underdog-saluki-27b-iq2-mix-2026-11-10T02-09-59-228Z-9zyu/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/metrics.json": "6b040ddfcb49322d376bc8ceaec8a08353d44e411178b91dd4329f7e68650cb1", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-docblocks-theme-roundtrip/docblocks-theme-roundtrip/docblocks-theme-roundtrip-underdog-saluki-27b-iq2-mix-2026-11-10T02-43-21-395Z-w8ls/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/metrics.json": "3737f397d71e3c5a8ecccb325d230dec9bc975c2c6627f870394a0ff33456186", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-powerpoint-deck/craftbook-powerpoint-deck/craftbook-powerpoint-deck-underdog-saluki-27b-iq2-mix-2026-11-10T03-05-12-731Z-u76b/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/metrics.json": "2945978b2e1eb8b9d3414e53df37665744ac3dfd12eda4a75386d18ef5929c82", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-wikipedia-research-brief/wikipedia-research-brief/wikipedia-research-brief-underdog-saluki-27b-iq2-mix-2026-11-10T04-37-26-781Z-v7zl/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/metrics.json": "bb1e051cdf27856666098c9369476ec787696267e65994a6955c002fff833931", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-week-plan/craftbook-week-plan/craftbook-week-plan-underdog-saluki-27b-iq2-mix-2026-11-10T05-31-35-469Z-qnfz/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/metrics.json": "1f7a579a5ac69346509aedc6a7626edaf7bde04eab7b919f5b4c84c22ed8d942", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-ab-test-readout/craftbook-ab-test-readout/craftbook-ab-test-readout-underdog-saluki-27b-iq2-mix-2026-11-10T06-57-26-407Z-3wb1/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/metrics.json": "9c495d4563dc4be50e03583f07b278ac0747f796e1fcba937f228804e2cd8e3f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-annotated-bibliography/craftbook-annotated-bibliography/craftbook-annotated-bibliography-underdog-saluki-27b-iq2-mix-2026-11-10T08-08-39-484Z-f8cd/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/metrics.json": "b2a4d18d1556be94a70f8eb1da1bf01971d2aa9a04fd76229e25ab85e34054cb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-productivity-craftbook-research-to-document/craftbook-research-to-document/craftbook-research-to-document-underdog-saluki-27b-iq2-mix-2026-11-10T08-53-56-539Z-295e/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/metrics.json": "5d7a0dfc725429d63ba1dbc6fa99b488ff39f20d294c7cfe464bdb5d99e25787", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-fix-squisq-bugs/fix-squisq-bugs/fix-squisq-bugs-underdog-saluki-27b-iq2-mix-2026-11-10T09-40-09-307Z-no9t/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/metrics.json": "ac7dc679ce2da569fd30ad00dcbaccec84fb79c841d28117bf2501daea5868a6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-dev-craftbook-routing/dev-craftbook-routing/dev-craftbook-routing-underdog-saluki-27b-iq2-mix-2026-11-10T10-06-52-675Z-5gvb/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/metrics.json": "595d1912f4af60f97c64de33da4422deb080dc5cf48d672f4308e7370cd8d527", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-interface-contract/interface-contract/interface-contract-underdog-saluki-27b-iq2-mix-2026-11-10T10-13-02-629Z-xihj/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/metrics.json": "6e32451dfb68e0502bb95b7a53e14f10960aaa33e778724e2eb17d492b6b4b92", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-code-review/craftbook-code-review/craftbook-code-review-underdog-saluki-27b-iq2-mix-2026-11-10T10-29-58-625Z-kelb/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/metrics.json": "853b2f89c6f3dd8f00313634e2ca27bcea0ebf47ac269267ede784eb33fccff7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-api-contract-review/craftbook-api-contract-review/craftbook-api-contract-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-00-03-208Z-gfja/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/metrics.json": "95d969856d02428d3b513303fe56214b88ab43e18caa864249a33b344e7d4e00", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-deep-security-review/craftbook-deep-security-review/craftbook-deep-security-review-underdog-saluki-27b-iq2-mix-2026-11-10T11-56-39-229Z-cava/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/metrics.json": "c0f0a7d9530bee6b462b091845e52bdc635a1de986d7f938f58a8f2b1bce7a29", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-large-pr-review/large-pr-review/large-pr-review-underdog-saluki-27b-iq2-mix-2026-11-10T12-50-20-834Z-9d29/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/metrics.json": "df95149da194b8faf207e3b94858c4892cbcae4d55810886fe19f205de1cb856", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-codemod-sweep/craftbook-codemod-sweep/craftbook-codemod-sweep-underdog-saluki-27b-iq2-mix-2026-11-10T13-17-14-802Z-tdv3/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/metrics.json": "48d359d210b823a283d116ffb0b401d49fcfa54c477c89b4ed01afbb1e1dd4cb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-refactor-module/craftbook-refactor-module/craftbook-refactor-module-underdog-saluki-27b-iq2-mix-2026-11-10T14-28-09-481Z-ilo4/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/metrics.json": "2ad4338157927aceb90133e797607d540c34fd37a1336f1fbc4956c3a7d26afa", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-developer-craftbook-bug-fix-tdd/craftbook-bug-fix-tdd/craftbook-bug-fix-tdd-underdog-saluki-27b-iq2-mix-2026-11-10T15-19-58-344Z-ilty/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq/metrics.json": "c7ab4c13a5189f408b2b4806bf538887aae929b0cac41a730536e2729641f60a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-find-vs-create/craftbook-find-vs-create/craftbook-find-vs-create-underdog-saluki-27b-iq2-mix-2026-11-10T16-22-59-365Z-ikxq/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/metrics.json": "188044c15ec2a2b56f2f8aff0935fe203b95428e1b039e9f4994267f25b654eb", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-edit-midtask/craftbook-edit-midtask/craftbook-edit-midtask-underdog-saluki-27b-iq2-mix-2026-11-10T16-28-41-233Z-pq67/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/metrics.json": "027806f127c9a4efeb5e5f09e6b89708c1fb0f1fc7e2577470a8145e360be7ac", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-invoice-run/craftbook-invoice-run/craftbook-invoice-run-underdog-saluki-27b-iq2-mix-2026-11-10T16-55-17-811Z-d5ml/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/metrics.json": "71c50611e7b34757f925a248f435ef70206e4421bfa887085d60674dce0aa55d", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-linear/craftbook-author-linear/craftbook-author-linear-underdog-saluki-27b-iq2-mix-2026-11-10T17-52-16-268Z-99gm/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/metrics.json": "1b41c3307902f042312ef1c07dc72c4473eab409ac2ab9c162c5bd2adef91368", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-route-multi/craftbook-route-multi/craftbook-route-multi-underdog-saluki-27b-iq2-mix-2026-11-10T18-52-14-365Z-dhdr/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/metrics.json": "2a8238f943d5ec5cad08ab127c19317b338f04a5760017fecc281c7ac5ff328b", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-export-generalize/craftbook-export-generalize/craftbook-export-generalize-underdog-saluki-27b-iq2-mix-2026-11-10T19-54-06-807Z-50yi/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/metrics.json": "887a99f3d07653829d9a2656164b8410a8d47ac8c6220bd9020769f1eba6a03f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-params/craftbook-author-params/craftbook-author-params-underdog-saluki-27b-iq2-mix-2026-11-10T22-28-51-912Z-754f/host.json": "55a8277d1a10b255eefb4e74231b9a200c5cbffbecdaa5870d3a16b436091ea7", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-interruption-2026-11-09.json": "68f351e9914e78a2dc814ad8f7a61249548323474a019ad8f7041743b5bdfa1f", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/fanout-remainder/craftbook-author-fanout/craftbook-author-fanout-underdog-saluki-27b-iq2-mix-2026-11-06T19-54-43-767Z-b8ob/postmortem.md": "2577bc502db17776cf1d23177d83147a03d04af360c0dfc00a0febe4383f5828", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3-interruption-2026-11-11.json": "a6a8feef6b47c4c64ce54b0c65b9959f352fecc24513e07eb9fcad982ff95ea6", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/workspace/data/inventory.json": "2a6f44cb0bb8d193c2abf527264e828fbf882e8a744180a786e784fe0688b439", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/tasks/.next-id": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/tasks/1/about.md": "3c867fa42d31f4d570a598de66feafe3811a7565cc504650419129ead3075c19", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/tasks/1/notes.jsonl": "f35b8bf699206a53d1e3713d67c3adcd4d664ee05f79c30978e1585126b4d08a", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/projects/inventory-health-check/tasks/1/task.json": "cd102aa962b3a216b1e4a1d5c1f93798a03d7de28898ddacbbfb7ecfea86ea26", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/craftbook-templates/in/inventory-health-report/manifest.json": "67db4a076a840a0d0669b33b1ccecdcb0f65cba7c5fca4dd3a477a9052414f39", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/craftbook-templates/in/inventory-health-report/versions/1.0.0/about.md": "0c752e963323cb006706b5f6579ba57cb3e5a0b4dd2597b82cbbe10f57569dc8", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/craftbook-templates/in/inventory-health-report/versions/1.0.0/manifest.json": "132d805a0f3e275ec9ae22360ca40c5b171151fae0381c0bd0659edcb6d275a9", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/advisory-interruption-2026-11-11/craftbook-templates/in/inventory-health-report/versions/1.0.0/scripts/inventory-health-gate.ts": "bf5519238090215ccd010d863160505c2390ac76e81870170b9069b6dcd0d130", "evals/runs/saluki-native-0.1.57-2026-11-03/continuation-2026-11-06/round3/r1-saluki-cuda-complex-work-craftbook-author-gate-script/craftbook-author-gate-script/craftbook-author-gate-script-underdog-saluki-27b-iq2-mix-2026-11-11T00-06-24-814Z-1294/postmortem.md": "bcff598fa30017d3b840076a4a7bc1346b6de974f47ee2c396d57068cd03a94b", "evals/runs/saluki-native-0.1.57-2026-11-03/final-historical-comparisons.json": "eada650438ae7983036a909c54b3f6af637ea2e5bc1a4639365d69ffed640712", "evals/runs/saluki-native-0.1.57-2026-11-03/final-report.md": "479a2d797daf9873b49df33b0e7fec8116932742179c012abb999ce3451ae8df", "evals/runs/saluki-native-0.1.57-2026-11-03/model-scorecard.md": "b8127518af6e1ef2ad47e6a70ad9cf475020cce8d715cddcc57791136605edd2" } } }